From d84210022ea9d80f8060e8d05b38d68555485b94 Mon Sep 17 00:00:00 2001 From: Pratyksh Gupta Date: Sat, 6 Dec 2025 12:38:59 +0530 Subject: [PATCH 001/132] Fix #1249: Correct REINDEX INDEX syntax to use single index instead of concatenating all --- src/sqlancer/postgres/gen/PostgresReindexGenerator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index 9bb5ec5cd..dc0d2cf34 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -1,7 +1,6 @@ package sqlancer.postgres.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -39,7 +38,7 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { if (indexes.isEmpty()) { throw new IgnoreMeException(); } - sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining())); + sb.append(Randomly.fromList(indexes).getIndexName()); break; case TABLE: sb.append("TABLE "); From 05bb28ce7a6f8e1152293fc8d080a94f9e4dc480 Mon Sep 17 00:00:00 2001 From: Aman Date: Sun, 7 Dec 2025 12:33:03 +0530 Subject: [PATCH 002/132] Add Spark support for TLP Oracle --- .github/workflows/main.yml | 38 ++ pom.xml | 9 +- src/sqlancer/Main.java | 2 + src/sqlancer/spark/SparkErrors.java | 67 ++++ src/sqlancer/spark/SparkGlobalState.java | 11 + src/sqlancer/spark/SparkOptions.java | 43 +++ src/sqlancer/spark/SparkProvider.java | 122 +++++++ src/sqlancer/spark/SparkSchema.java | 114 ++++++ src/sqlancer/spark/SparkToStringVisitor.java | 120 +++++++ .../spark/ast/SparkBetweenOperation.java | 10 + .../spark/ast/SparkBinaryOperation.java | 11 + .../spark/ast/SparkCaseOperation.java | 13 + .../spark/ast/SparkCastOperation.java | 25 ++ .../spark/ast/SparkColumnReference.java | 11 + src/sqlancer/spark/ast/SparkConstant.java | 194 ++++++++++ src/sqlancer/spark/ast/SparkExpression.java | 7 + src/sqlancer/spark/ast/SparkFunction.java | 13 + src/sqlancer/spark/ast/SparkInOperation.java | 12 + src/sqlancer/spark/ast/SparkJoin.java | 46 +++ src/sqlancer/spark/ast/SparkOrderingTerm.java | 10 + src/sqlancer/spark/ast/SparkSelect.java | 42 +++ .../spark/ast/SparkTableReference.java | 13 + .../spark/ast/SparkUnaryPostfixOperation.java | 13 + .../spark/ast/SparkUnaryPrefixOperation.java | 12 + .../spark/gen/SparkExpressionGenerator.java | 336 ++++++++++++++++++ .../spark/gen/SparkInsertGenerator.java | 47 +++ .../spark/gen/SparkTableGenerator.java | 100 ++++++ test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestSparkTLP.java | 20 ++ 29 files changed, 1460 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/spark/SparkErrors.java create mode 100644 src/sqlancer/spark/SparkGlobalState.java create mode 100644 src/sqlancer/spark/SparkOptions.java create mode 100644 src/sqlancer/spark/SparkProvider.java create mode 100644 src/sqlancer/spark/SparkSchema.java create mode 100644 src/sqlancer/spark/SparkToStringVisitor.java create mode 100644 src/sqlancer/spark/ast/SparkBetweenOperation.java create mode 100644 src/sqlancer/spark/ast/SparkBinaryOperation.java create mode 100644 src/sqlancer/spark/ast/SparkCaseOperation.java create mode 100644 src/sqlancer/spark/ast/SparkCastOperation.java create mode 100644 src/sqlancer/spark/ast/SparkColumnReference.java create mode 100644 src/sqlancer/spark/ast/SparkConstant.java create mode 100644 src/sqlancer/spark/ast/SparkExpression.java create mode 100644 src/sqlancer/spark/ast/SparkFunction.java create mode 100644 src/sqlancer/spark/ast/SparkInOperation.java create mode 100644 src/sqlancer/spark/ast/SparkJoin.java create mode 100644 src/sqlancer/spark/ast/SparkOrderingTerm.java create mode 100644 src/sqlancer/spark/ast/SparkSelect.java create mode 100644 src/sqlancer/spark/ast/SparkTableReference.java create mode 100644 src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java create mode 100644 src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java create mode 100644 src/sqlancer/spark/gen/SparkExpressionGenerator.java create mode 100644 src/sqlancer/spark/gen/SparkInsertGenerator.java create mode 100644 src/sqlancer/spark/gen/SparkTableGenerator.java create mode 100644 test/sqlancer/dbms/TestSparkTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c53192aa..41bd92d3c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -333,6 +333,44 @@ jobs: - name: Run Tests run: HIVE_AVAILABLE=true mvn -Dtest=TestHiveTLP test + spark: + name: DBMS Tests (Spark) + runs-on: ubuntu-latest + + services: + spark: + image: apache/spark:3.5.1 + ports: + - 10000:10000 + + command: >- + /opt/spark/bin/spark-submit + --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 + --name "Thrift JDBC/ODBC Server" + --master local[*] + --driver-memory 4g + --conf spark.hive.server2.thrift.port=10000 + --conf spark.sql.warehouse.dir=/tmp/spark-warehouse + spark-internal + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + + - name: Build SQLancer + run: mvn -B package -DskipTests=true + + - name: Run Tests + run: SPARK_AVAILABLE=true mvn -Dtest=TestSparkTLP test + hsqldb: name: DBMS Tests (HSQLB) runs-on: ubuntu-latest diff --git a/pom.xml b/pom.xml index 2037b71ce..c7a38c9aa 100644 --- a/pom.xml +++ b/pom.xml @@ -329,7 +329,7 @@ org.slf4j - slf4j-simple + slf4j-simple 2.0.6 @@ -381,7 +381,7 @@ org.apache.hive hive-jdbc - 4.0.1 + 3.1.2 org.apache.hive @@ -393,6 +393,11 @@ hive-cli 4.0.1 + + org.apache.hadoop + hadoop-common + 3.2.4 + diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 1f2642f95..f778bd7da 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -48,6 +48,7 @@ import sqlancer.tidb.TiDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.spark.SparkProvider; public final class Main { @@ -756,6 +757,7 @@ private static void checkForIssue799(List> providers) providers.add(new DuckDBProvider()); providers.add(new H2Provider()); providers.add(new HiveProvider()); + providers.add(new SparkProvider()); providers.add(new HSQLDBProvider()); providers.add(new MariaDBProvider()); providers.add(new MaterializeProvider()); diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java new file mode 100644 index 000000000..97c8056a3 --- /dev/null +++ b/src/sqlancer/spark/SparkErrors.java @@ -0,0 +1,67 @@ +package sqlancer.spark; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class SparkErrors { + + private SparkErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("cannot resolve"); + errors.add("AnalysisException"); + errors.add("data type mismatch"); + errors.add("undefined function"); + errors.add("mismatched input"); + errors.add("due to data type mismatch"); + + // --- Invalid Literals + errors.add("The value of the typed literal"); + + errors.add("DATATYPE_MISMATCH"); + errors.add("cannot be cast to"); + + errors.add("Overflow"); + errors.add("Divide by zero"); // Common if spark.sql.ansi.enabled is true + errors.add("division by zero"); + + // --- Group By / Aggregation errors --- + errors.add("grouping expressions"); + errors.add("expression is neither present in the group by"); + errors.add("is not a valid grouping expression"); + errors.add("is not contained in either an aggregate function or the GROUP BY clause"); + errors.add("PARSE_SYNTAX_ERROR"); + errors.add("Syntax error"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("not enough data columns"); + errors.add("cannot write to"); + errors.add("incompatible types"); + errors.add("too many data columns"); + errors.add("cannot be cast to"); + errors.add("Error running query"); + errors.add("The value of the typed literal"); + errors.add("Cannot safely cast"); // Found in logs: Decimal -> Date + errors.add("AnalysisException"); // Spark throws this for almost all insert failures + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkGlobalState.java b/src/sqlancer/spark/SparkGlobalState.java new file mode 100644 index 000000000..e79826332 --- /dev/null +++ b/src/sqlancer/spark/SparkGlobalState.java @@ -0,0 +1,11 @@ +package sqlancer.spark; + +import sqlancer.SQLGlobalState; + +public class SparkGlobalState extends SQLGlobalState { + + @Override + protected SparkSchema readSchema() throws Exception { + return SparkSchema.fromConnection(getConnection(), getDatabaseName()); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkOptions.java b/src/sqlancer/spark/SparkOptions.java new file mode 100644 index 000000000..c9422a910 --- /dev/null +++ b/src/sqlancer/spark/SparkOptions.java @@ -0,0 +1,43 @@ +package sqlancer.spark; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.spark.gen.SparkExpressionGenerator; + +@Parameters(separators = "=", commandDescription = "Spark SQL (default port: " + SparkOptions.DEFAULT_PORT + + ", default host: " + SparkOptions.DEFAULT_HOST + ")") +public class SparkOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 10000; + + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(SparkOracleFactory.TLPWhere); + + public enum SparkOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(SparkGlobalState globalState) throws SQLException { + SparkExpressionGenerator gen = new SparkExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SparkErrors.getExpressionErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }; + } + + @Override + public List getTestOracleFactory() { + return oracle; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java new file mode 100644 index 000000000..f53ca10a8 --- /dev/null +++ b/src/sqlancer/spark/SparkProvider.java @@ -0,0 +1,122 @@ +package sqlancer.spark; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.spark.gen.SparkInsertGenerator; +import sqlancer.spark.gen.SparkTableGenerator; + +@AutoService(DatabaseProvider.class) +public class SparkProvider extends SQLProviderAdapter { + + public SparkProvider() { + super(SparkGlobalState.class, SparkOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(SparkInsertGenerator::getQuery); // You will need to create this class + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(SparkGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(SparkGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + default: + throw new AssertionError(a); + } + } + + @Override + public void generateDatabase(SparkGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + String tableName = globalState.getSchema().getFreeTableName(); + SQLQueryAdapter qt = SparkTableGenerator.generate(globalState, tableName); + success = globalState.executeStatement(qt); + } while (!success); + } + + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + SparkProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + + if (host == null) { + host = SparkOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = SparkOptions.DEFAULT_PORT; + } + + String databaseName = globalState.getDatabaseName(); + + // Spark uses the Hive driver for JDBC usually + String url = String.format("jdbc:hive2://%s:%d/%s", host, port, "default"); + + // Connect to default to create the fuzzing DB + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + con.close(); + + // Connect to the specific fuzzing DB + con = DriverManager.getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName), username, + password); + try (Statement s = con.createStatement()) { + // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the generator loves to do. + s.execute("SET spark.sql.ansi.enabled=false"); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "spark"; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java new file mode 100644 index 000000000..849652b19 --- /dev/null +++ b/src/sqlancer/spark/SparkSchema.java @@ -0,0 +1,114 @@ +package sqlancer.spark; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.spark.SparkSchema.SparkTable; + +public class SparkSchema extends AbstractSchema { + + public enum SparkDataType { + STRING, INTEGER, DOUBLE, BOOLEAN, TIMESTAMP, DATE; + + public static SparkDataType getRandomType() { + return Randomly.fromList(Arrays.asList(values())); + } + } + + public static class SparkColumn extends AbstractTableColumn { + public SparkColumn(String name, SparkTable table, SparkDataType type) { + super(name, table, type); + } + } + + public static class SparkTables extends AbstractTables { + public SparkTables(List tables) { + super(tables); + } + } + + public static class SparkTable extends AbstractRelationalTable { + public SparkTable(String name, List columns, boolean isView) { + super(name, columns, Collections.emptyList(), isView); + } + } + + public SparkSchema(List databaseTables) { + super(databaseTables); + } + + public static SparkSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.toLowerCase().startsWith("v"); + SparkTable t = new SparkTable(tableName, databaseColumns, isView); + for (SparkColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new SparkSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet tableRs = s.executeQuery("SHOW TABLES"); + while (tableRs.next()) { + // Spark SHOW TABLES output: database, tableName, isTemporary + String tableName = tableRs.getString("tableName"); + tableNames.add(tableName); + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("DESCRIBE %s", tableName))) { + while (rs.next()) { + String columnName = rs.getString("col_name"); + String dataType = rs.getString("data_type"); + // Filter out Spark partition info or comments usually at bottom of describe + if (columnName.startsWith("#") || columnName.isEmpty()) + continue; + + columns.add(new SparkColumn(columnName, null, getColumnType(dataType))); + } + } + } + return columns; + } + + private static SparkDataType getColumnType(String typeString) { + String upper = typeString.toUpperCase(); + if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) + return SparkDataType.STRING; + if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) + return SparkDataType.INTEGER; + if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) + return SparkDataType.DOUBLE; + if (upper.startsWith("BOOLEAN")) + return SparkDataType.BOOLEAN; + if (upper.startsWith("TIMESTAMP")) + return SparkDataType.TIMESTAMP; + if (upper.startsWith("DATE")) + return SparkDataType.DATE; + return SparkDataType.STRING; // Fallback + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkToStringVisitor.java b/src/sqlancer/spark/SparkToStringVisitor.java new file mode 100644 index 000000000..91f47e32c --- /dev/null +++ b/src/sqlancer/spark/SparkToStringVisitor.java @@ -0,0 +1,120 @@ +package sqlancer.spark; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.spark.ast.SparkCastOperation; +import sqlancer.spark.ast.SparkConstant; +import sqlancer.spark.ast.SparkExpression; +import sqlancer.spark.ast.SparkJoin; +import sqlancer.spark.ast.SparkSelect; + +public class SparkToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(SparkExpression expr) { + if (expr instanceof SparkConstant) { + visit((SparkConstant) expr); + } else if (expr instanceof SparkSelect) { + visit((SparkSelect) expr); + } else if (expr instanceof SparkJoin) { + visit((SparkJoin) expr); + } else if (expr instanceof SparkCastOperation) { + visit((SparkCastOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(SparkConstant constant) { + sb.append(constant.toString()); + } + + private void visit(SparkSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + // Spark supports OFFSET, though strictly usually with LIMIT or in newer versions + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + private void visit(SparkJoin join) { + switch (join.getJoinType()) { + case INNER: + sb.append(" INNER JOIN "); + break; + case LEFT_OUTER: + sb.append(" LEFT JOIN "); + break; + case RIGHT_OUTER: + sb.append(" RIGHT JOIN "); + break; + case FULL_OUTER: + sb.append(" FULL JOIN "); + break; + case LEFT_SEMI: + sb.append(" LEFT SEMI JOIN "); + break; + // Spark also supports LEFT ANTI, which Hive might lack in some older versions + case LEFT_ANTI: + sb.append(" LEFT ANTI JOIN "); + break; + case CROSS: + sb.append(" CROSS JOIN "); + break; + default: + throw new UnsupportedOperationException("Join type not supported in Spark visitor: " + join.getJoinType()); + } + visit((TableReferenceNode) join.getRightTable()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + private void visit(SparkCastOperation cast) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + sb.append(cast.getType()); + sb.append(")"); + } + + public static String asString(SparkExpression expr) { + SparkToStringVisitor visitor = new SparkToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkBetweenOperation.java b/src/sqlancer/spark/ast/SparkBetweenOperation.java new file mode 100644 index 000000000..f229c1c7c --- /dev/null +++ b/src/sqlancer/spark/ast/SparkBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class SparkBetweenOperation extends NewBetweenOperatorNode implements SparkExpression { + + public SparkBetweenOperation(SparkExpression left, SparkExpression middle, SparkExpression right, boolean isTrue) { + super(left, middle, right, isTrue); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkBinaryOperation.java b/src/sqlancer/spark/ast/SparkBinaryOperation.java new file mode 100644 index 000000000..04af0ec4c --- /dev/null +++ b/src/sqlancer/spark/ast/SparkBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class SparkBinaryOperation extends NewBinaryOperatorNode implements SparkExpression { + + public SparkBinaryOperation(SparkExpression left, SparkExpression right, Operator op) { + super(left, right, op); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkCaseOperation.java b/src/sqlancer/spark/ast/SparkCaseOperation.java new file mode 100644 index 000000000..fb1ee0cd8 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkCaseOperation.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class SparkCaseOperation extends NewCaseOperatorNode implements SparkExpression { + + public SparkCaseOperation(SparkExpression switchCondition, List conditions, + List expressions, SparkExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkCastOperation.java b/src/sqlancer/spark/ast/SparkCastOperation.java new file mode 100644 index 000000000..3bc5eb30d --- /dev/null +++ b/src/sqlancer/spark/ast/SparkCastOperation.java @@ -0,0 +1,25 @@ +package sqlancer.spark.ast; + +import sqlancer.spark.SparkSchema.SparkDataType; + +public class SparkCastOperation implements SparkExpression { + + private final SparkExpression expression; + private final SparkDataType type; + + public SparkCastOperation(SparkExpression expression, SparkDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + public SparkExpression getExpression() { + return expression; + } + + public SparkDataType getType() { + return type; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkColumnReference.java b/src/sqlancer/spark/ast/SparkColumnReference.java new file mode 100644 index 000000000..75e92d267 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.spark.SparkSchema.SparkColumn; + +public class SparkColumnReference extends ColumnReferenceNode implements SparkExpression { + + public SparkColumnReference(SparkColumn column) { + super(column); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkConstant.java b/src/sqlancer/spark/ast/SparkConstant.java new file mode 100644 index 000000000..9f73af59f --- /dev/null +++ b/src/sqlancer/spark/ast/SparkConstant.java @@ -0,0 +1,194 @@ +package sqlancer.spark.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public abstract class SparkConstant implements SparkExpression { + + public boolean isNull() { + return false; + } + + public static class SparkNullConstant extends SparkConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public String toString() { + return "NULL"; + } + } + + public static class SparkIntConstant extends SparkConstant { + + private final long value; + + public SparkIntConstant(long value) { + this.value = value; + } + + public long getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class SparkDoubleConstant extends SparkConstant { + + private final double value; + + public SparkDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "CAST('Infinity' AS DOUBLE)"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "CAST('-Infinity' AS DOUBLE)"; + } else if (Double.isNaN(value)) { + return "CAST('NaN' AS DOUBLE)"; + } + return String.valueOf(value); + } + } + + public static class SparkDecimalConstant extends SparkConstant { + + private final BigDecimal value; + + public SparkDecimalConstant(BigDecimal value) { + this.value = value; + } + + public BigDecimal getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class SparkTimestampConstant extends SparkConstant { + + private final String textRepr; + + public SparkTimestampConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); // Spark prefers full timestamp + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + } + + public static class SparkDateConstant extends SparkConstant { + + private final String textRepr; + + public SparkDateConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + } + + public static class SparkStringConstant extends SparkConstant { + + private final String value; + + public SparkStringConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''").replace("\\", "\\\\") + "'"; + } + } + + public static class SparkBooleanConstant extends SparkConstant { + + private final boolean value; + + public SparkBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static SparkConstant createNullConstant() { + return new SparkNullConstant(); + } + + public static SparkConstant createIntConstant(long value) { + return new SparkIntConstant(value); + } + + public static SparkConstant createDoubleConstant(double value) { + return new SparkDoubleConstant(value); + } + + public static SparkConstant createDecimalConstant(BigDecimal value) { + return new SparkDecimalConstant(value); + } + + public static SparkConstant createTimestampConstant(long value) { + return new SparkTimestampConstant(value); + } + + public static SparkConstant createDateConstant(long value) { + return new SparkDateConstant(value); + } + + public static SparkConstant createStringConstant(String value) { + return new SparkStringConstant(value); + } + + public static SparkConstant createBooleanConstant(boolean value) { + return new SparkBooleanConstant(value); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkExpression.java b/src/sqlancer/spark/ast/SparkExpression.java new file mode 100644 index 000000000..a130096e3 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkExpression.java @@ -0,0 +1,7 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.spark.SparkSchema.SparkColumn; + +public interface SparkExpression extends Expression { +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkFunction.java b/src/sqlancer/spark/ast/SparkFunction.java new file mode 100644 index 000000000..d5740ee36 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkFunction.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class SparkFunction extends NewFunctionNode implements SparkExpression { + + public SparkFunction(List args, F func) { + super(args, func); + } + +} diff --git a/src/sqlancer/spark/ast/SparkInOperation.java b/src/sqlancer/spark/ast/SparkInOperation.java new file mode 100644 index 000000000..37a80e3ff --- /dev/null +++ b/src/sqlancer/spark/ast/SparkInOperation.java @@ -0,0 +1,12 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class SparkInOperation extends NewInOperatorNode implements SparkExpression { + + public SparkInOperation(SparkExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkJoin.java b/src/sqlancer/spark/ast/SparkJoin.java new file mode 100644 index 000000000..44da7fba4 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkJoin.java @@ -0,0 +1,46 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.Join; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; + +public class SparkJoin implements SparkExpression, Join { + + private final SparkTableReference leftTable; + private final SparkTableReference rightTable; + private final JoinType joinType; + private SparkExpression onClause; + + public enum JoinType { + INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, LEFT_ANTI, CROSS; + } + + public SparkJoin(SparkTableReference leftTable, SparkTableReference rightTable, JoinType joinType, + SparkExpression onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onClause = onClause; + } + + public SparkTableReference getLeftTable() { + return leftTable; + } + + public SparkTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public SparkExpression getOnClause() { + return onClause; + } + + @Override + public void setOnClause(SparkExpression onClause) { + this.onClause = onClause; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkOrderingTerm.java b/src/sqlancer/spark/ast/SparkOrderingTerm.java new file mode 100644 index 000000000..824801c00 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkOrderingTerm.java @@ -0,0 +1,10 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class SparkOrderingTerm extends NewOrderingTerm implements SparkExpression { + + public SparkOrderingTerm(SparkExpression expr, Ordering ordering) { + super(expr, ordering); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkSelect.java b/src/sqlancer/spark/ast/SparkSelect.java new file mode 100644 index 000000000..0986ce0a6 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkSelect.java @@ -0,0 +1,42 @@ +package sqlancer.spark.ast; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkSelect extends SelectBase + implements Select, SparkExpression { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (SparkExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (SparkJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return SparkToStringVisitor.asString(this); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkTableReference.java b/src/sqlancer/spark/ast/SparkTableReference.java new file mode 100644 index 000000000..92a59ad3d --- /dev/null +++ b/src/sqlancer/spark/ast/SparkTableReference.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.spark.SparkSchema; + +public class SparkTableReference extends TableReferenceNode + implements SparkExpression { + + public SparkTableReference(SparkSchema.SparkTable table) { + super(table); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java new file mode 100644 index 000000000..f1082a655 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class SparkUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements SparkExpression { + + public SparkUnaryPostfixOperation(SparkExpression expr, Operator op) { + super(expr, op); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java new file mode 100644 index 000000000..d1bd94ab4 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class SparkUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements SparkExpression { + + public SparkUnaryPrefixOperation(SparkExpression expr, Operator op) { + super(expr, op); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/gen/SparkExpressionGenerator.java b/src/sqlancer/spark/gen/SparkExpressionGenerator.java new file mode 100644 index 000000000..faf8a07f0 --- /dev/null +++ b/src/sqlancer/spark/gen/SparkExpressionGenerator.java @@ -0,0 +1,336 @@ +package sqlancer.spark.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkDataType; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.ast.SparkBetweenOperation; +import sqlancer.spark.ast.SparkBinaryOperation; +import sqlancer.spark.ast.SparkCaseOperation; +import sqlancer.spark.ast.SparkCastOperation; +import sqlancer.spark.ast.SparkColumnReference; +import sqlancer.spark.ast.SparkConstant; +import sqlancer.spark.ast.SparkExpression; +import sqlancer.spark.ast.SparkFunction; +import sqlancer.spark.ast.SparkInOperation; +import sqlancer.spark.ast.SparkJoin; +import sqlancer.spark.ast.SparkOrderingTerm; +import sqlancer.spark.ast.SparkSelect; +import sqlancer.spark.ast.SparkTableReference; +import sqlancer.spark.ast.SparkUnaryPostfixOperation; +import sqlancer.spark.ast.SparkUnaryPrefixOperation; + +public class SparkExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { + + private final SparkGlobalState globalState; + private List tables; + + private enum Expression { + UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, CAST, FUNC, BETWEEN, IN, + CASE; + } + + public SparkExpressionGenerator(SparkGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public SparkExpression negatePredicate(SparkExpression predicate) { + return new SparkUnaryPrefixOperation(predicate, SparkUnaryPrefixOperator.NOT); + } + + @Override + public SparkExpression isNull(SparkExpression expr) { + return new SparkUnaryPostfixOperation(expr, SparkUnaryPostfixOperator.IS_NULL); + } + + @Override + protected SparkExpression generateExpression(int depth) { + return generateExpressionInternal(depth); + } + + private SparkExpression generateExpressionInternal(int depth) throws AssertionError { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBooleanWithRatherLowProbability()) { + allowAggregates = false; // aggregate function calls cannot be nested + SparkAggregateFunction aggregate = SparkAggregateFunction.getRandom(); + return new SparkFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + + switch (expr) { + case UNARY_PREFIX: + return new SparkUnaryPrefixOperation(generateExpression(depth + 1), SparkUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new SparkUnaryPostfixOperation(generateExpression(depth + 1), SparkUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = SparkBinaryComparisonOperator.getRandom(); + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = SparkBinaryLogicalOperator.getRandom(); + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + SparkBinaryArithmeticOperator.getRandom()); + case CAST: + return new SparkCastOperation(generateExpression(depth + 1), SparkDataType.getRandomType()); + case FUNC: + SparkFunc func = SparkFunc.getRandom(); + return new SparkFunction<>(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new SparkBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new SparkInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new SparkCaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(expr); + } + } + + @Override + public SparkExpression generateConstant() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return SparkConstant.createNullConstant(); + } + SparkDataType[] values = SparkDataType.values(); + SparkDataType constantType = Randomly.fromOptions(values); + switch (constantType) { + case STRING: + return SparkConstant.createStringConstant(globalState.getRandomly().getString()); + case INTEGER: + return SparkConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DOUBLE: + return SparkConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BOOLEAN: + return SparkConstant.createBooleanConstant(Randomly.getBoolean()); + case TIMESTAMP: + return SparkConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case DATE: + return SparkConstant.createDateConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(constantType); + } + } + + @Override + protected SparkExpression generateColumn() { + SparkColumn column = Randomly.fromList(columns); + return new SparkColumnReference(column); + } + + @Override + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (SparkExpression curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new SparkOrderingTerm(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + } + + @Override + public SparkExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + return this; + } + + @Override + public SparkExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public SparkSelect generateSelect() { + return new SparkSelect(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new SparkTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean allowAggregates) { + if (Randomly.getBoolean()) { + return List.of(new SparkColumnReference(new SparkColumn("*", null, null))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new SparkColumnReference(c)) + .collect(Collectors.toList()); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + public enum SparkUnaryPrefixOperator implements Operator { + NOT("NOT"), PLUS("+"), MINUS("-"), BITWISE_NOT("~"); + + private String textRepr; + + SparkUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkUnaryPostfixOperator implements Operator { + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + SparkUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryComparisonOperator implements Operator { + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), RLIKE("RLIKE"); + + private String textRepr; + + SparkBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryLogicalOperator implements Operator { + AND("AND"), OR("OR"); + + private String textRepr; + + SparkBinaryLogicalOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryArithmeticOperator implements Operator { + // Spark supports || for concat, and bitwise operators &, |, ^ + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), BITWISE_AND("&"), BITWISE_OR("|"), + BITWISE_XOR("^"); + + private String textRepr; + + SparkBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkAggregateFunction { + COUNT(1), SUM(1), AVG(1), MIN(1), MAX(1), VARIANCE(1), VAR_SAMP(1), STDDEV_POP(1), STDDEV_SAMP(1), COVAR_POP(2), + COVAR_SAMP(2), CORR(2); + + private int nrArgs; + + SparkAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static SparkAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + public enum SparkFunc { + ROUND(2), FLOOR(1), ABS(1), CEIL(1); + + private int nrArgs; + private boolean isVariadic; + + SparkFunc(int nrArgs) { + this(nrArgs, false); + } + + SparkFunc(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static SparkFunc getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java new file mode 100644 index 000000000..29232fdb2 --- /dev/null +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -0,0 +1,47 @@ +package sqlancer.spark.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.spark.SparkErrors; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkInsertGenerator extends AbstractInsertGenerator { + + private final SparkGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + private final SparkExpressionGenerator gen; + + public SparkInsertGenerator(SparkGlobalState globalState) { + this.globalState = globalState; + this.gen = new SparkExpressionGenerator(globalState); + } + + public static SQLQueryAdapter getQuery(SparkGlobalState globalState) { + return new SparkInsertGenerator(globalState).generate(); + } + + @Override + protected void insertValue(SparkColumn column) { + sb.append(SparkToStringVisitor.asString(gen.generateConstant())); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + SparkTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + + sb.append(" VALUES "); + + List columns = table.getColumns(); + insertColumns(columns); + + SparkErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java new file mode 100644 index 000000000..68cafdafb --- /dev/null +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -0,0 +1,100 @@ +package sqlancer.spark.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.spark.SparkErrors; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkDataType; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkTableGenerator { + + private enum ColumnConstraints { + NOT_NULL, DEFAULT + // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources (Parquet/ORC) + // without specific catalogs (like Delta/Iceberg), so we limit to constraints Spark SQL widely accepts. + } + + private final SparkGlobalState globalState; + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + private final SparkExpressionGenerator gen; + private final SparkTable table; + private final List columnsToBeAdded = new ArrayList<>(); + + public SparkTableGenerator(SparkGlobalState globalState, String tableName) { + this.tableName = tableName; + this.globalState = globalState; + this.table = new SparkTable(tableName, columnsToBeAdded, false); + this.gen = new SparkExpressionGenerator(globalState).setColumns(columnsToBeAdded); + } + + public static SQLQueryAdapter generate(SparkGlobalState globalState, String tableName) { + SparkTableGenerator generator = new SparkTableGenerator(globalState, tableName); + return generator.create(); + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + + sb.append("CREATE TABLE "); + sb.append(globalState.getDatabaseName()); + sb.append("."); + sb.append(tableName); + sb.append(" ("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(i); + } + sb.append(")"); + sb.append(" USING PARQUET"); + + // TODO: implement PARTITION BY clause + // TODO: implement CLUSTERED BY clauses + // TODO: implement ROW FORMAT and STORED AS clauses + // TODO: randomly add some predefined TABLEPROPERTIES + + SparkErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + + private void appendColumn(int columnId) { + String columnName = DBMSCommon.createColumnName(columnId); + sb.append(columnName); + sb.append(" "); + SparkDataType randType = SparkSchema.SparkDataType.getRandomType(); + sb.append(randType); + columnsToBeAdded.add(new SparkColumn(columnName, table, randType)); + appendColumnConstraint(); + } + + private void appendColumnConstraint() { + if (Randomly.getBoolean()) { + return; + } + + ColumnConstraints constraint = Randomly.fromOptions(ColumnConstraints.values()); + switch (constraint) { + case NOT_NULL: + sb.append(" NOT NULL"); + break; + case DEFAULT: + sb.append(" DEFAULT "); + sb.append(SparkToStringVisitor.asString(gen.generateConstant())); + sb.append(" "); + break; + default: + throw new AssertionError(constraint); + } + } +} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index f5aeefa12..f6be45648 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -11,6 +11,7 @@ public class TestConfig { public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String HIVE_ENV = "HIVE_AVAILABLE"; + public static final String SPARK_ENV = "SPARK_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; public static final String MYSQL_ENV = "MYSQL_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestSparkTLP.java b/test/sqlancer/dbms/TestSparkTLP.java new file mode 100644 index 000000000..83302ceff --- /dev/null +++ b/test/sqlancer/dbms/TestSparkTLP.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSparkTLP { + + @Test + public void testSparkTLPWhere() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.SPARK_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", + "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "spark", "--oracle", "TLPWhere" })); + } +} \ No newline at end of file From b4c3763d9db5b179e41c170dfd5d7fa31d4bc9eb Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 10:48:47 +0000 Subject: [PATCH 003/132] Fix where clause in Materialize --- src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java index d61e452f8..f7ff76305 100644 --- a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -600,6 +600,7 @@ public String generateOptimizedQueryString(MaterializeSelect select, Materialize } select.setSelectType(SelectType.ALL); } + select.setWhereClause(whereCondition); return select.asString(); } From e4e013d064232ea7f9d17b8bdbb5fb2411c035ae Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 10:55:54 +0000 Subject: [PATCH 004/132] Fix up Materialize CI --- .github/workflows/main.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c53192aa..026fa52bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -392,10 +392,7 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest - sleep 5 - # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 - psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -420,10 +417,7 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest - sleep 5 - # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 - psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest - name: Set up JDK 11 uses: actions/setup-java@v3 with: From dacd33c3ba671bed063d618d864c14d5bba0a4f5 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 12:50:10 +0000 Subject: [PATCH 005/132] Try to fix duplicate table creation in Materialize --- src/sqlancer/materialize/MaterializeProvider.java | 11 ++++++++--- .../materialize/gen/MaterializeTableGenerator.java | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java index 5b18b1dfb..e7bdb4c4f 100644 --- a/src/sqlancer/materialize/MaterializeProvider.java +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -240,12 +240,17 @@ protected void readFunctions(MaterializeGlobalState globalState) throws SQLExcep } protected void createTables(MaterializeGlobalState globalState, int numTables) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < numTables) { + int existingTables = globalState.getSchema().getDatabaseTables().size(); + int createdTables = 0; + int nextTableIndex = existingTables; + while (existingTables + createdTables < numTables) { try { - String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + String tableName = DBMSCommon.createTableName(nextTableIndex++); SQLQueryAdapter createTable = MaterializeTableGenerator.generate(tableName, globalState.getSchema(), generateOnlyKnown, globalState); - globalState.executeStatement(createTable); + if (globalState.executeStatement(createTable)) { + createdTables++; + } } catch (IgnoreMeException e) { } diff --git a/src/sqlancer/materialize/gen/MaterializeTableGenerator.java b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java index f132f7370..c6772db47 100644 --- a/src/sqlancer/materialize/gen/MaterializeTableGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java @@ -52,6 +52,7 @@ public MaterializeTableGenerator(String tableName, MaterializeSchema newSchema, errors.add("no collation was derived for partition key column"); errors.add("inherits from generated column but specifies identity"); errors.add("inherits from generated column but specifies default"); + errors.add("already exists"); MaterializeCommon.addCommonExpressionErrors(errors); MaterializeCommon.addCommonTableErrors(errors); } From 44a156e8fe7c9c913c782808ce0540beca3be342 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 14:10:22 +0000 Subject: [PATCH 006/132] Adapt materialize insert error message --- src/sqlancer/materialize/gen/MaterializeInsertGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java index 01015514b..7a5374b95 100644 --- a/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java @@ -26,7 +26,7 @@ public static SQLQueryAdapter insert(MaterializeGlobalState globalState) { MaterializeCommon.addCommonExpressionErrors(errors); errors.add("multiple assignments to same column"); errors.add("violates foreign key constraint"); - errors.add("value too long for type character varying"); + errors.add("value too long for type character"); errors.add("conflicting key value violates exclusion constraint"); errors.add("violates not-null constraint"); errors.add("current transaction is aborted"); From e039cbaa71e0dfabc69e816bf5714692f5c5e412 Mon Sep 17 00:00:00 2001 From: Elshaarawy-1 Date: Wed, 11 Feb 2026 11:36:08 +0200 Subject: [PATCH 007/132] Add DuckDBSchema.getIndexes() implementation Based on PR #1171 Addresses #1163 --- src/sqlancer/duckdb/DuckDBSchema.java | 23 ++++++++++++++++--- .../duckdb/gen/DuckDBIndexGenerator.java | 4 ++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 379576fd8..7e89da7e6 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -218,8 +218,8 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { public static class DuckDBTable extends AbstractRelationalTable { - public DuckDBTable(String tableName, List columns, boolean isView) { - super(tableName, columns, Collections.emptyList(), isView); + public DuckDBTable(String tableName, List columns, List indexes, boolean isView) { + super(tableName, columns, indexes, isView); } } @@ -233,7 +233,8 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName } List databaseColumns = getTableColumns(con, tableName); boolean isView = tableName.startsWith("v"); - DuckDBTable t = new DuckDBTable(tableName, databaseColumns, isView); + List indexes = getIndexes(con, tableName, databaseName); + DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { c.setTable(t); } @@ -243,6 +244,22 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName return new DuckDBSchema(databaseTables); } + private static List getIndexes(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT INDEX_NAME FROM duckdb_indexes() WHERE DATABASE_NAME = '%s' and TABLE_NAME = '%s';", + databaseName, tableName))) { + while (rs.next()) { + String indexName = rs.getString("INDEX_NAME"); + indexes.add(TableIndex.create(indexName)); + } + } + } + return indexes; + } + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 597fda19f..cc4114f63 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -23,7 +23,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append("UNIQUE "); } sb.append("INDEX "); - sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); @@ -40,7 +40,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } } sb.append(")"); - errors.add("already exists!"); + // errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("cannot create an index on the rowid"); } From 353edac5b4217070dd37206925c3a5823f8f62f0 Mon Sep 17 00:00:00 2001 From: Elshaarawy-1 Date: Wed, 11 Feb 2026 11:36:33 +0200 Subject: [PATCH 008/132] Fix database name filter in getIndexes() Remove DATABASE_NAME filter to fix index collision bug. --- src/sqlancer/duckdb/DuckDBSchema.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 7e89da7e6..8141220fa 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import sqlancer.IgnoreMeException; @@ -233,7 +232,7 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName } List databaseColumns = getTableColumns(con, tableName); boolean isView = tableName.startsWith("v"); - List indexes = getIndexes(con, tableName, databaseName); + List indexes = getIndexes(con, tableName); DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { c.setTable(t); @@ -244,13 +243,12 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName return new DuckDBSchema(databaseTables); } - private static List getIndexes(SQLConnection con, String tableName, String databaseName) - throws SQLException { + private static List getIndexes(SQLConnection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format( - "SELECT INDEX_NAME FROM duckdb_indexes() WHERE DATABASE_NAME = '%s' and TABLE_NAME = '%s';", - databaseName, tableName))) { + "SELECT index_name FROM duckdb_indexes() WHERE database_name = current_database() AND table_name = '%s';", + tableName))) { while (rs.next()) { String indexName = rs.getString("INDEX_NAME"); indexes.add(TableIndex.create(indexName)); @@ -259,7 +257,7 @@ private static List getIndexes(SQLConnection con, String tableName, } return indexes; } - + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { From 5caba27dacd843f4dd70a69f4febfc35e94d97e8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 1 Mar 2026 05:02:22 +0100 Subject: [PATCH 009/132] ClickHouse: add EMPTY_LIST_OF_COLUMNS_PASSED to expected errors ClickHouse PR https://github.com/ClickHouse/ClickHouse/pull/81835 added a check that forbids creating tables without insertable columns (e.g., tables with only MATERIALIZED or ALIAS columns). Since SQLancer's random table generator can produce such definitions, ClickHouse now returns EMPTY_LIST_OF_COLUMNS_PASSED, which should be treated as an expected error rather than causing an AssertionError. The table creation retry loop in ClickHouseProvider.generateDatabase already handles expected errors by regenerating the table definition. --- src/sqlancer/clickhouse/ClickHouseErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 2c4d8d1b8..09fbe5ea8 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -41,6 +41,7 @@ public static List getExpectedExpressionErrors() { "Cannot convert out of range floating point value to integer type", "Unexpected inf or nan to integer conversion", "No such name in Block::erase", // https://github.com/ClickHouse/ClickHouse/issues/42769 "EMPTY_LIST_OF_COLUMNS_QUERIED", // https://github.com/ClickHouse/ClickHouse/issues/43003 + "EMPTY_LIST_OF_COLUMNS_PASSED", // https://github.com/ClickHouse/ClickHouse/pull/81835 "cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)", "AMBIGUOUS_IDENTIFIER", "CYCLIC_ALIASES", "Positional argument numeric constant expression is not representable as", "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", From 1506cf28b48f632d29df76dd992d85f0dfa1d829 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Mar 2026 22:40:19 +0000 Subject: [PATCH 010/132] Bump jacoco version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2037b71ce..767fedcce 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ org.jacoco jacoco-maven-plugin - 0.8.8 + 0.8.12 From d7518297ead5d46bff05552f8ecedd042afe5058 Mon Sep 17 00:00:00 2001 From: albertZhangTJ Date: Tue, 3 Mar 2026 01:37:32 +0000 Subject: [PATCH 011/132] Add expected errors for postgres 18 --- src/sqlancer/postgres/gen/PostgresCommon.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 63bc885cb..992cbc880 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -48,6 +48,7 @@ public static List getCommonTableErrors() { errors.add("is not commutative"); // exclude errors.add("operator requires run-time type coercion"); // exclude + errors.add("partitioned tables cannot be unlogged"); return errors; } @@ -59,6 +60,8 @@ public static void addCommonTableErrors(ExpectedErrors errors) { public static List getCommonExpressionErrors() { ArrayList errors = new ArrayList<>(); + errors.add("for encoding \"SQL_ASCII\" does not exist"); + errors.add("invalid byte sequence for encoding"); errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); From d7b272b93b946a66e39dc64067d5ee5b21b105f5 Mon Sep 17 00:00:00 2001 From: albertZhangTJ Date: Tue, 3 Mar 2026 02:14:40 +0000 Subject: [PATCH 012/132] Update postgres version in CI to 18 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c53192aa..c07452653 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -504,7 +504,7 @@ jobs: - name: Set up PostgreSQL uses: harmon758/postgresql-action@v1.0.0 with: - postgresql version: '13' + postgresql version: '18' postgresql user: 'sqlancer' postgresql password: 'sqlancer' postgresql db: 'test' From a4bf0a9d65c52843d05634ca21c72f1cb1eeea72 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Sun, 22 Mar 2026 18:39:58 +0000 Subject: [PATCH 013/132] Disable SET_UNLOGGED_LOGGED for partitioned table for PG18; Fix NPE for PostgresBinaryComparisonOperation --- src/sqlancer/postgres/PostgresSchema.java | 24 ++++++++++++++++--- .../PostgresBinaryComparisonOperation.java | 2 +- .../gen/PostgresAlterTableGenerator.java | 3 +++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index c99c8648e..17089b011 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -164,6 +164,7 @@ public enum TableType { private final TableType tableType; private final List statistics; private final boolean isInsertable; + private final boolean isPartitioned; public PostgresTable(String tableName, List columns, List indexes, TableType tableType, List statistics, boolean isView, boolean isInsertable) { @@ -171,6 +172,18 @@ public PostgresTable(String tableName, List columns, List columns, List indexes, + TableType tableType, List statistics, boolean isView, boolean isInsertable, + boolean isPartitioned) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + this.isPartitioned = isPartitioned; } public List getStatistics() { @@ -185,6 +198,10 @@ public boolean isInsertable() { return isInsertable; } + public boolean isPartitioned() { + return isPartitioned; + } + } public static final class PostgresStatisticsObject { @@ -225,11 +242,12 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery( - "SELECT table_name, table_schema, table_type, is_insertable_into FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { + "SELECT t.table_name, t.table_schema, t.table_type, t.is_insertable_into, c.relkind FROM information_schema.tables t JOIN pg_class c ON c.relname = t.table_name JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = t.table_schema WHERE t.table_schema='public' OR t.table_schema LIKE 'pg_temp_%' ORDER BY t.table_name;")) { while (rs.next()) { String tableName = rs.getString("table_name"); String tableTypeSchema = rs.getString("table_schema"); boolean isInsertable = rs.getBoolean("is_insertable_into"); + boolean isPartitioned = "p".equals(rs.getString("relkind")); // TODO: also check insertable // TODO: insert into view? boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || @@ -240,7 +258,7 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa List indexes = getIndexes(con, tableName); List statistics = getStatistics(con); PostgresTable t = new PostgresTable(tableName, databaseColumns, indexes, tableType, statistics, - isView, isInsertable); + isView, isInsertable, isPartitioned); for (PostgresColumn c : databaseColumns) { c.setTable(t); } @@ -324,4 +342,4 @@ public String getDatabaseName() { return databaseName; } -} +} \ No newline at end of file diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 95efe5b72..daa464b77 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -126,7 +126,7 @@ public PostgresConstant getExpectedValue() { PostgresConstant leftExpectedValue = getLeft().getExpectedValue(); PostgresConstant rightExpectedValue = getRight().getExpectedValue(); if (leftExpectedValue == null || rightExpectedValue == null) { - return null; + return PostgresConstant.createNullConstant(); } return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); } diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 6e0c436ee..eb53f6df1 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -127,6 +127,9 @@ public List getActions(ExpectedErrors errors) { if (!randomTable.hasIndexes()) { action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); } + if (randomTable.isPartitioned()){ + action.remove(Action.SET_LOGGED_UNLOGGED); + } if (action.isEmpty()) { throw new IgnoreMeException(); } From 2b8796cda88cfa17856388ef57f1f8ea9989e6d6 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Sun, 22 Mar 2026 20:03:57 +0000 Subject: [PATCH 014/132] Add cannot drop inherited constraint expected error --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index eb53f6df1..d39006f3c 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -238,6 +238,8 @@ public SQLQueryAdapter generate() { errors.add("is in a primary key"); errors.add("is an identity column"); errors.add("is in index used as replica identity"); + // PG18 update: otherwise we need to encode contraint inheritance info in PostgreColumn + errors.add("cannot drop inherited constraint"); } break; case ALTER_COLUMN_SET_STATISTICS: From 21ec5c73f541a5cb85b488378caed188f184b240 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Mon, 23 Mar 2026 15:49:09 +0000 Subject: [PATCH 015/132] Format the previous commits; Update PostgresBinaryComparisonOperator null handling --- src/sqlancer/postgres/PostgresSchema.java | 2 +- .../postgres/ast/PostgresBinaryComparisonOperation.java | 3 ++- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 +- src/sqlancer/postgres/gen/PostgresCommon.java | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 17089b011..3cc8a6e4b 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -172,7 +172,7 @@ public PostgresTable(String tableName, List columns, List getActions(ExpectedErrors errors) { if (!randomTable.hasIndexes()) { action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); } - if (randomTable.isPartitioned()){ + if (randomTable.isPartitioned()) { action.remove(Action.SET_LOGGED_UNLOGGED); } if (action.isEmpty()) { diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 992cbc880..e9f27aeb2 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -61,7 +61,7 @@ public static List getCommonExpressionErrors() { ArrayList errors = new ArrayList<>(); errors.add("for encoding \"SQL_ASCII\" does not exist"); - errors.add("invalid byte sequence for encoding"); + errors.add("invalid byte sequence for encoding"); errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); From 9a3d65e3a10f974d9ccd4b125caaf05fba122c7f Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Mon, 23 Mar 2026 16:17:14 +0000 Subject: [PATCH 016/132] Fix checkStyle errors --- src/sqlancer/postgres/PostgresSchema.java | 3 ++- .../postgres/ast/PostgresBinaryComparisonOperation.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 3cc8a6e4b..17ef652dd 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -342,4 +342,5 @@ public String getDatabaseName() { return databaseName; } -} \ No newline at end of file +} + diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 9341d3578..3ee69ba17 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; -import sqlancer.Randomly; import sqlancer.IgnoreMeException; +import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -138,3 +138,4 @@ public PostgresDataType getExpressionType() { } } + From d4f459fcfb3127f609a59c71097f5c2ac4e48a27 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Mon, 23 Mar 2026 16:19:49 +0000 Subject: [PATCH 017/132] Fix checkStyle errors --- src/sqlancer/postgres/PostgresSchema.java | 1 - src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 17ef652dd..20337fea7 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -343,4 +343,3 @@ public String getDatabaseName() { } } - diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 3ee69ba17..b77060dfd 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -138,4 +138,3 @@ public PostgresDataType getExpressionType() { } } - From f2b42cef4fe10e8500cc4181f241b863f1bae1ea Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 26 Mar 2026 21:52:56 +0800 Subject: [PATCH 018/132] Fix Materialize CI by removing deprecated MZ_EAT_MY_DATA flag MZ_EAT_MY_DATA=1 is a deprecated env var from old Materialize versions that triggered an initialization path running CockroachDB-style SQL (SET CLUSTER SETTING sql.stats.forecasts.enabled = false), which is no longer valid in the current Materialize architecture. Also add a pg_isready health check to ensure Materialize is accepting connections before tests run. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3514d904..665394c8a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -392,7 +392,8 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest + until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -417,7 +418,8 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest + until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 uses: actions/setup-java@v3 with: From 708af09439398b662c674ce7752f954a4a0b1295 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 08:08:49 +0800 Subject: [PATCH 019/132] Replace hardcoded sleeps with readiness loops in CI workflows Use `until` polling loops instead of fixed `sleep` durations to wait for services to become ready, reducing flakiness and unnecessary wait time across CnosDB, ClickHouse, CockroachDB, OceanBase, Presto, TiDB, YugabyteDB, and Doris jobs. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 665394c8a..b0bd5355d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -111,7 +111,7 @@ jobs: run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - sleep 5 + until curl -sf http://127.0.0.1:8902/ping 2>/dev/null; do sleep 1; done - name: Run Tests run: | CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test @@ -137,7 +137,7 @@ jobs: run: | docker pull clickhouse/clickhouse-server:24.3.1.2672 docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:24.3.1.2672 - sleep 5 + until curl -sf http://127.0.0.1:8123/ping 2>/dev/null; do sleep 1; done - name: Run Tests run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test - name: Show fatal errors @@ -166,7 +166,7 @@ jobs: run: | wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & - sleep 10 + until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests @@ -194,7 +194,7 @@ jobs: run: | wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & - sleep 10 + until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests @@ -483,7 +483,7 @@ jobs: - name: Set up OceanBase run: | docker run -p 2881:2881 --name oceanbase-ce -e MODE=mini -d oceanbase/oceanbase-ce:4.2.1-lts - sleep 120 + until mysql -h127.1 -uroot@test -P2881 --connect-timeout=3 -Doceanbase -A -e "SELECT 1" 2>/dev/null; do sleep 5; done mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | @@ -537,13 +537,13 @@ jobs: docker pull prestodb/presto:latest echo "connector.name=memory" >> memory.properties docker run -p 8080:8080 -d -v ./memory.properties:/opt/presto-server/etc/catalog/memory.properties --name presto prestodb/presto:latest - sleep 30 + until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests run: | PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoNoREC test - docker restart presto && sleep 30 + docker restart presto && until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoTLP test sqlite: name: DBMS Tests (SQLite) @@ -607,9 +607,9 @@ jobs: run: | docker pull hawkingrei/tidb-playground:nightly-2025-09-16 docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 - sleep 10 + until mysql -h 127.0.0.1 -P 4000 -u root --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done - name: Create SQLancer user - run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: | TIDB_AVAILABLE=true mvn -Dtest=TestTiDBTLP test @@ -634,9 +634,9 @@ jobs: run: | docker pull hawkingrei/tidb-playground:nightly-2025-09-16 docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 - sleep 10 + until mysql -h 127.0.0.1 -P 4000 -u root --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done - name: Create SQLancer user - run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test @@ -659,7 +659,7 @@ jobs: run: | docker pull yugabytedb/yugabyte:latest docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false - sleep 5 + until pg_isready -h localhost -p 5433; do sleep 1; done - name: Run Tests run: | YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLNoREC test @@ -696,7 +696,7 @@ jobs: cd ../be ./bin/start_be.sh --daemon - sleep 30 + until mysql -u root -h 127.0.0.1 --port 9030 --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done IP=$(hostname -I | awk '{print $1}') mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';" mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" From 5032eae9a96fa2a37e8d6f0ff0f4f2a475a0e925 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 08:41:24 +0800 Subject: [PATCH 020/132] Update MySQL CI from 8.0.36 to 8.4 and fix compatibility issues - Update mysql service image in CI from 8.0.36 to 8.4 - Remove show_old_temporals system variable (removed in MySQL 8.4) - Update expected error for integer literals in ORDER BY: MySQL 8.4 changed the error context from 'order clause' to 'EXISTS subquery' Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 2 +- src/sqlancer/mysql/MySQLErrors.java | 2 +- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 1 - src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 665394c8a..34c0e7e5d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -438,7 +438,7 @@ jobs: runs-on: ubuntu-latest services: mysql: - image: mysql:8.0.36 + image: mysql:8.4 env: MYSQL_ROOT_PASSWORD: root ports: diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index f01b30e03..bec149920 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -32,7 +32,7 @@ public static List getExpressionRegexErrors() { // "00000000000000000000-0" } - errors.add(Pattern.compile("Unknown column '.*' in 'order clause'")); + errors.add(Pattern.compile("Unknown column '.*' in 'EXISTS subquery'")); return errors; } diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 79333eb36..e350685ef 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -99,7 +99,6 @@ private enum Action { SCHEMA_DEFINITION_CACHE("schema_definition_cache", (r) -> r.getLong(256, 524288), Scope.GLOBAL), // SHOW_CREATE_TABLE_VERBOSITY("show_create_table_verbosity", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - SHOW_OLD_TEMPORALS("show_old_temporals", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), /* * sort_buffer_size is commented out as a workaround for https://bugs.mysql.com/bug.php?id=95969 */ diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 28665c328..c613f2622 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -37,7 +37,7 @@ public class MySQLPivotedQuerySynthesisOracle public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { super(globalState); MySQLErrors.addExpressionErrors(errors); - errors.add("in 'order clause'"); // e.g., Unknown column '2067708013' in 'order clause' + errors.add("in 'EXISTS subquery'"); // e.g., Unknown column '2067708013' in 'EXISTS subquery' (MySQL 8.4+) } @Override From 227904af7a92ecfa69f5be07ddc8782fc7d3b2ef Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 09:01:12 +0800 Subject: [PATCH 021/132] Optimize CI: remove unnecessary fetch-depth and update action versions - Remove fetch-depth: 0 from all jobs (full git history is not needed for running tests) - Update actions/checkout from v3/v2 to v4 - Update actions/setup-java from v3 to v4 Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 156 +++++++++++++------------------------ 1 file changed, 52 insertions(+), 104 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 665394c8a..f0b8317e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,11 +24,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -48,11 +46,9 @@ jobs: name: DBMS Tests (Citus) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -96,11 +92,9 @@ jobs: name: DBMS Tests (CnosDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -122,11 +116,9 @@ jobs: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -151,11 +143,9 @@ jobs: name: DBMS Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -179,11 +169,9 @@ jobs: name: QPG Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -213,11 +201,9 @@ jobs: - 8000:8000 - 3307:3307 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -234,9 +220,7 @@ jobs: name: DBMS Tests (DataFusion) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Rust uses: actions-rs/toolchain@v1 with: @@ -247,7 +231,7 @@ jobs: cd src/sqlancer/datafusion/server/datafusion_server cargo run & sleep 300 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -263,11 +247,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -283,11 +265,9 @@ jobs: name: DBMS Tests (H2) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -319,11 +299,9 @@ jobs: volumes: - warehouse:/opt/hive/data/warehouse steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -337,11 +315,9 @@ jobs: name: DBMS Tests (HSQLB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -366,11 +342,9 @@ jobs: options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -386,16 +360,14 @@ jobs: name: DBMS Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Materialize run: | docker pull materialize/materialized:latest docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -412,16 +384,14 @@ jobs: name: QPG Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Materialize run: | docker pull materialize/materialized:latest docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -445,11 +415,9 @@ jobs: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -469,11 +437,9 @@ jobs: name: DBMS Tests (OceanBase) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -494,9 +460,7 @@ jobs: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up PostgreSQL uses: harmon758/postgresql-action@v1.0.0 with: @@ -505,7 +469,7 @@ jobs: postgresql password: 'sqlancer' postgresql db: 'test' - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -523,11 +487,9 @@ jobs: name: DBMS Tests (Presto) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -550,11 +512,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -573,11 +533,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -592,11 +550,9 @@ jobs: name: DBMS Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -619,11 +575,9 @@ jobs: name: QPG Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -644,11 +598,9 @@ jobs: name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -671,11 +623,9 @@ jobs: name: DBMS Tests (Apache Doris) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -713,11 +663,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' From 91d7a6ae05158c1d908a1b416b08bceafb5011d0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 09:03:37 +0800 Subject: [PATCH 022/132] Restore 'order clause' error alongside 'EXISTS subquery' for MySQL 8.4 MySQL 8.4 uses 'order clause' when the ORDER BY only contains an integer literal, and 'EXISTS subquery' when ORDER BY also contains an EXISTS expression. Both patterns need to be handled. Co-Authored-By: Claude Sonnet 4.6 --- src/sqlancer/mysql/MySQLErrors.java | 1 + src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index bec149920..989c8fed6 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -32,6 +32,7 @@ public static List getExpressionRegexErrors() { // "00000000000000000000-0" } + errors.add(Pattern.compile("Unknown column '.*' in 'order clause'")); errors.add(Pattern.compile("Unknown column '.*' in 'EXISTS subquery'")); return errors; diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index c613f2622..c1fe893b6 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -37,6 +37,7 @@ public class MySQLPivotedQuerySynthesisOracle public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { super(globalState); MySQLErrors.addExpressionErrors(errors); + errors.add("in 'order clause'"); // e.g., Unknown column '2067708013' in 'order clause' errors.add("in 'EXISTS subquery'"); // e.g., Unknown column '2067708013' in 'EXISTS subquery' (MySQL 8.4+) } From 8c143a3d4951bf85d891a0fbbefcfa0f08ff38d9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 09:09:47 +0800 Subject: [PATCH 023/132] Fix CnosDB readiness check to use TCP port probe The /ping endpoint does not exist on CnosDB, causing curl -f to loop forever. Switch to nc -z which just checks that the port is open. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0bd5355d..6bd47d3e1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -111,7 +111,7 @@ jobs: run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - until curl -sf http://127.0.0.1:8902/ping 2>/dev/null; do sleep 1; done + until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done - name: Run Tests run: | CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test From 6837dd1d05e7da8ef49a577a6b58498c03da9325 Mon Sep 17 00:00:00 2001 From: Carmen Kwan Date: Thu, 2 Apr 2026 01:12:40 +0200 Subject: [PATCH 024/132] Fix CONTRIBUTING.md Stop referencing files that don't exist --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5db3bdc69..ea5baea1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ If you do not find an option to import Maven projects, you might need to install ## Implementing Support for a New DBMS -The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see `DuckDBNoRECOracle` or `DuckDBQueryPartitioningWhereTester` for TLP). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). +The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see enum value `NOREC` in `DuckDBOracleFactory`). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). Please consider the following suggestions when creating a PR to contribute a new DBMS: * Ensure that `mvn verify -DskipTests=true` does not result in style violations. From 8787427efc005358f83913ace3fc1beaf2350c2e Mon Sep 17 00:00:00 2001 From: Aman Bihari <161295600+codebreaker32@users.noreply.github.com> Date: Fri, 3 Apr 2026 06:16:57 +0000 Subject: [PATCH 025/132] Remove outdated comment and wrong errors --- src/sqlancer/spark/SparkErrors.java | 2 -- src/sqlancer/spark/SparkProvider.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java index 97c8056a3..83ba84bb3 100644 --- a/src/sqlancer/spark/SparkErrors.java +++ b/src/sqlancer/spark/SparkErrors.java @@ -35,8 +35,6 @@ public static List getExpressionErrors() { errors.add("expression is neither present in the group by"); errors.add("is not a valid grouping expression"); errors.add("is not contained in either an aggregate function or the GROUP BY clause"); - errors.add("PARSE_SYNTAX_ERROR"); - errors.add("Syntax error"); return errors; } diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java index f53ca10a8..89f24a738 100644 --- a/src/sqlancer/spark/SparkProvider.java +++ b/src/sqlancer/spark/SparkProvider.java @@ -28,7 +28,7 @@ public SparkProvider() { } public enum Action implements AbstractAction { - INSERT(SparkInsertGenerator::getQuery); // You will need to create this class + INSERT(SparkInsertGenerator::getQuery); private final SQLQueryProvider sqlQueryProvider; From 787fcf3225a313059c92dadf3cab25ea6cbc120e Mon Sep 17 00:00:00 2001 From: Aman Date: Sat, 4 Apr 2026 15:05:35 +0530 Subject: [PATCH 026/132] Format spark/* --- src/sqlancer/Main.java | 2 +- src/sqlancer/spark/SparkErrors.java | 2 +- src/sqlancer/spark/SparkGlobalState.java | 2 +- src/sqlancer/spark/SparkOptions.java | 2 +- src/sqlancer/spark/SparkProvider.java | 5 ++-- src/sqlancer/spark/SparkSchema.java | 24 ++++++++++++------- src/sqlancer/spark/SparkToStringVisitor.java | 5 ++-- .../spark/ast/SparkBetweenOperation.java | 2 +- .../spark/ast/SparkBinaryOperation.java | 2 +- .../spark/ast/SparkCaseOperation.java | 2 +- .../spark/ast/SparkCastOperation.java | 2 +- .../spark/ast/SparkColumnReference.java | 2 +- src/sqlancer/spark/ast/SparkConstant.java | 2 +- src/sqlancer/spark/ast/SparkExpression.java | 2 +- src/sqlancer/spark/ast/SparkInOperation.java | 2 +- src/sqlancer/spark/ast/SparkJoin.java | 2 +- src/sqlancer/spark/ast/SparkOrderingTerm.java | 2 +- src/sqlancer/spark/ast/SparkSelect.java | 2 +- .../spark/ast/SparkTableReference.java | 2 +- .../spark/ast/SparkUnaryPostfixOperation.java | 2 +- .../spark/ast/SparkUnaryPrefixOperation.java | 2 +- .../spark/gen/SparkExpressionGenerator.java | 2 +- .../spark/gen/SparkInsertGenerator.java | 2 +- .../spark/gen/SparkTableGenerator.java | 8 ++++--- 24 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index f778bd7da..faf35e3c9 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -44,11 +44,11 @@ import sqlancer.postgres.PostgresProvider; import sqlancer.presto.PrestoProvider; import sqlancer.questdb.QuestDBProvider; +import sqlancer.spark.SparkProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; import sqlancer.yugabyte.ysql.YSQLProvider; -import sqlancer.spark.SparkProvider; public final class Main { diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java index 83ba84bb3..a3a96061f 100644 --- a/src/sqlancer/spark/SparkErrors.java +++ b/src/sqlancer/spark/SparkErrors.java @@ -62,4 +62,4 @@ public static List getInsertErrors() { public static void addInsertErrors(ExpectedErrors errors) { errors.addAll(getInsertErrors()); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkGlobalState.java b/src/sqlancer/spark/SparkGlobalState.java index e79826332..d78c737e5 100644 --- a/src/sqlancer/spark/SparkGlobalState.java +++ b/src/sqlancer/spark/SparkGlobalState.java @@ -8,4 +8,4 @@ public class SparkGlobalState extends SQLGlobalState protected SparkSchema readSchema() throws Exception { return SparkSchema.fromConnection(getConnection(), getDatabaseName()); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkOptions.java b/src/sqlancer/spark/SparkOptions.java index c9422a910..7b347ceef 100644 --- a/src/sqlancer/spark/SparkOptions.java +++ b/src/sqlancer/spark/SparkOptions.java @@ -40,4 +40,4 @@ public TestOracle create(SparkGlobalState globalState) throws public List getTestOracleFactory() { return oracle; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java index 89f24a738..817a92471 100644 --- a/src/sqlancer/spark/SparkProvider.java +++ b/src/sqlancer/spark/SparkProvider.java @@ -109,7 +109,8 @@ public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLExce con = DriverManager.getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName), username, password); try (Statement s = con.createStatement()) { - // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the generator loves to do. + // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the + // generator loves to do. s.execute("SET spark.sql.ansi.enabled=false"); } return new SQLConnection(con); @@ -119,4 +120,4 @@ public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLExce public String getDBMSName() { return "spark"; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java index 849652b19..8d988e31e 100644 --- a/src/sqlancer/spark/SparkSchema.java +++ b/src/sqlancer/spark/SparkSchema.java @@ -85,8 +85,9 @@ private static List getTableColumns(SQLConnection con, String table String columnName = rs.getString("col_name"); String dataType = rs.getString("data_type"); // Filter out Spark partition info or comments usually at bottom of describe - if (columnName.startsWith("#") || columnName.isEmpty()) + if (columnName.startsWith("#") || columnName.isEmpty()) { continue; + } columns.add(new SparkColumn(columnName, null, getColumnType(dataType))); } @@ -97,18 +98,25 @@ private static List getTableColumns(SQLConnection con, String table private static SparkDataType getColumnType(String typeString) { String upper = typeString.toUpperCase(); - if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) + if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) { return SparkDataType.STRING; - if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) + } + if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) { return SparkDataType.INTEGER; - if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) + } + if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) { return SparkDataType.DOUBLE; - if (upper.startsWith("BOOLEAN")) + } + if (upper.startsWith("BOOLEAN")) { return SparkDataType.BOOLEAN; - if (upper.startsWith("TIMESTAMP")) + } + if (upper.startsWith("TIMESTAMP")) { return SparkDataType.TIMESTAMP; - if (upper.startsWith("DATE")) + } + if (upper.startsWith("DATE")) { return SparkDataType.DATE; + } return SparkDataType.STRING; // Fallback } -} \ No newline at end of file + +} diff --git a/src/sqlancer/spark/SparkToStringVisitor.java b/src/sqlancer/spark/SparkToStringVisitor.java index 91f47e32c..0777c86a6 100644 --- a/src/sqlancer/spark/SparkToStringVisitor.java +++ b/src/sqlancer/spark/SparkToStringVisitor.java @@ -63,7 +63,8 @@ private void visit(SparkSelect select) { sb.append(" LIMIT "); visit(select.getLimitClause()); } - // Spark supports OFFSET, though strictly usually with LIMIT or in newer versions + // Spark supports OFFSET, though strictly usually with LIMIT or in newer + // versions if (select.getOffsetClause() != null) { sb.append(" OFFSET "); visit(select.getOffsetClause()); @@ -117,4 +118,4 @@ public static String asString(SparkExpression expr) { visitor.visit(expr); return visitor.get(); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkBetweenOperation.java b/src/sqlancer/spark/ast/SparkBetweenOperation.java index f229c1c7c..59297ba8f 100644 --- a/src/sqlancer/spark/ast/SparkBetweenOperation.java +++ b/src/sqlancer/spark/ast/SparkBetweenOperation.java @@ -7,4 +7,4 @@ public class SparkBetweenOperation extends NewBetweenOperatorNode public SparkBinaryOperation(SparkExpression left, SparkExpression right, Operator op) { super(left, right, op); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkCaseOperation.java b/src/sqlancer/spark/ast/SparkCaseOperation.java index fb1ee0cd8..995fd7f52 100644 --- a/src/sqlancer/spark/ast/SparkCaseOperation.java +++ b/src/sqlancer/spark/ast/SparkCaseOperation.java @@ -10,4 +10,4 @@ public SparkCaseOperation(SparkExpression switchCondition, List List expressions, SparkExpression elseExpr) { super(switchCondition, conditions, expressions, elseExpr); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkCastOperation.java b/src/sqlancer/spark/ast/SparkCastOperation.java index 3bc5eb30d..547551285 100644 --- a/src/sqlancer/spark/ast/SparkCastOperation.java +++ b/src/sqlancer/spark/ast/SparkCastOperation.java @@ -22,4 +22,4 @@ public SparkExpression getExpression() { public SparkDataType getType() { return type; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkColumnReference.java b/src/sqlancer/spark/ast/SparkColumnReference.java index 75e92d267..ccd1b7855 100644 --- a/src/sqlancer/spark/ast/SparkColumnReference.java +++ b/src/sqlancer/spark/ast/SparkColumnReference.java @@ -8,4 +8,4 @@ public class SparkColumnReference extends ColumnReferenceNode { -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkInOperation.java b/src/sqlancer/spark/ast/SparkInOperation.java index 37a80e3ff..430d9b5c2 100644 --- a/src/sqlancer/spark/ast/SparkInOperation.java +++ b/src/sqlancer/spark/ast/SparkInOperation.java @@ -9,4 +9,4 @@ public class SparkInOperation extends NewInOperatorNode impleme public SparkInOperation(SparkExpression left, List right, boolean isNegated) { super(left, right, isNegated); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkJoin.java b/src/sqlancer/spark/ast/SparkJoin.java index 44da7fba4..a59eaff48 100644 --- a/src/sqlancer/spark/ast/SparkJoin.java +++ b/src/sqlancer/spark/ast/SparkJoin.java @@ -43,4 +43,4 @@ public SparkExpression getOnClause() { public void setOnClause(SparkExpression onClause) { this.onClause = onClause; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkOrderingTerm.java b/src/sqlancer/spark/ast/SparkOrderingTerm.java index 824801c00..870c8239b 100644 --- a/src/sqlancer/spark/ast/SparkOrderingTerm.java +++ b/src/sqlancer/spark/ast/SparkOrderingTerm.java @@ -7,4 +7,4 @@ public class SparkOrderingTerm extends NewOrderingTerm implemen public SparkOrderingTerm(SparkExpression expr, Ordering ordering) { super(expr, ordering); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkSelect.java b/src/sqlancer/spark/ast/SparkSelect.java index 0986ce0a6..8b59f5513 100644 --- a/src/sqlancer/spark/ast/SparkSelect.java +++ b/src/sqlancer/spark/ast/SparkSelect.java @@ -39,4 +39,4 @@ public String asString() { return SparkToStringVisitor.asString(this); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkTableReference.java b/src/sqlancer/spark/ast/SparkTableReference.java index 92a59ad3d..5bcbb5d03 100644 --- a/src/sqlancer/spark/ast/SparkTableReference.java +++ b/src/sqlancer/spark/ast/SparkTableReference.java @@ -10,4 +10,4 @@ public SparkTableReference(SparkSchema.SparkTable table) { super(table); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java index f1082a655..3dd9d28e2 100644 --- a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java +++ b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java @@ -10,4 +10,4 @@ public SparkUnaryPostfixOperation(SparkExpression expr, Operator op) { super(expr, op); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java index d1bd94ab4..5c1a8e4c6 100644 --- a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java +++ b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java @@ -9,4 +9,4 @@ public SparkUnaryPrefixOperation(SparkExpression expr, Operator op) { super(expr, op); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/gen/SparkExpressionGenerator.java b/src/sqlancer/spark/gen/SparkExpressionGenerator.java index faf8a07f0..3708f314a 100644 --- a/src/sqlancer/spark/gen/SparkExpressionGenerator.java +++ b/src/sqlancer/spark/gen/SparkExpressionGenerator.java @@ -333,4 +333,4 @@ public int getNrArgs() { } } } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java index 29232fdb2..b1755a848 100644 --- a/src/sqlancer/spark/gen/SparkInsertGenerator.java +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -44,4 +44,4 @@ private SQLQueryAdapter generate() { SparkErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, false, false); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java index 68cafdafb..937e52248 100644 --- a/src/sqlancer/spark/gen/SparkTableGenerator.java +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -19,8 +19,10 @@ public class SparkTableGenerator { private enum ColumnConstraints { NOT_NULL, DEFAULT - // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources (Parquet/ORC) - // without specific catalogs (like Delta/Iceberg), so we limit to constraints Spark SQL widely accepts. + // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources + // (Parquet/ORC) + // without specific catalogs (like Delta/Iceberg), so we limit to constraints + // Spark SQL widely accepts. } private final SparkGlobalState globalState; @@ -97,4 +99,4 @@ private void appendColumnConstraint() { throw new AssertionError(constraint); } } -} \ No newline at end of file +} From 22d5c1335d705fb9992ce347728ce90b7423a1cb Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:04:14 +0800 Subject: [PATCH 027/132] Fix DataFusion CI: replace blind sleep with build step and readiness poll The DataFusion CI job was flaky because it backgrounded `cargo run` with a fixed 300s sleep that raced against compilation time. Split into explicit build, start, and readiness-check steps so tests only run once the server is actually listening on port 50051. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 34c0e7e5d..d22c46ff9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -242,10 +242,14 @@ jobs: with: toolchain: stable override: true - - name: Run DataFusion Server + - name: Build DataFusion Server run: | cd src/sqlancer/datafusion/server/datafusion_server - cargo run & sleep 300 + cargo build + - name: Start DataFusion Server + run: | + cd src/sqlancer/datafusion/server/datafusion_server + cargo run & - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -254,6 +258,18 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true + - name: Wait for DataFusion Server + run: | + for i in $(seq 1 30); do + if nc -z 127.0.0.1 50051 2>/dev/null; then + echo "DataFusion server is ready" + exit 0 + fi + echo "Waiting for DataFusion server... ($i/30)" + sleep 10 + done + echo "DataFusion server failed to start within 300s" + exit 1 - name: Run Tests run: | DATAFUSION_AVAILABLE=true mvn test -Pdatafusion-tests From 9d26d70264ea85379830ec8a6907321ccbea6be1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:12:16 +0800 Subject: [PATCH 028/132] Pin chrono <0.4.40 to fix arrow-arith 52.2.0 build conflict chrono 0.4.40+ added `Datelike::quarter()` which conflicts with `ChronoDateExt::quarter()` in arrow-arith 52.2.0, causing ambiguous method resolution. Pin chrono below 0.4.40 until arrow dependencies are upgraded. Co-Authored-By: Claude Sonnet 4.6 --- src/sqlancer/datafusion/server/datafusion_server/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml index cd8b85e1d..332a88e30 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml +++ b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml @@ -16,7 +16,7 @@ arrow-schema = { version = "52.1.0", default-features = false } arrow-string = { version = "52.1.0", default-features = false } async-trait = "0.1.73" bytes = "1.4" -chrono = { version = "0.4.34", default-features = false } +chrono = { version = ">=0.4.34, <0.4.40", default-features = false } dashmap = "5.5.0" # This version is for SQLancer CI run datafusion = { version = "40.0.0" } From c548689d99bc3d9f5290d2ca28ec5965964c1405 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:32:02 +0800 Subject: [PATCH 029/132] Fix DataFusion server memory leak: clear state on database reset The server's DashMaps (contexts, statements, results) were never cleaned up between SQLancer rounds, causing unbounded memory growth that eventually stalled query throughput to 0/s. Co-Authored-By: Claude Sonnet 4.6 --- src/sqlancer/datafusion/server/datafusion_server/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs index 13ec73e96..057c34883 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs +++ b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs @@ -215,6 +215,11 @@ impl FlightSqlService for FlightSqlServiceImpl { let mut ctx_guard = self.ctx.lock().await; // Use `lock()` for async Mutex *ctx_guard = new_ctx; + + // Clear leaked state from previous round + self.statements.clear(); + self.results.clear(); + self.contexts.clear(); } // no authentication actually takes place here // see Ballista implementation for example of basic auth From 5b4f606ad4a9c83ff638e4329b8b53144651f738 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:43:34 +0800 Subject: [PATCH 030/132] Fix HSQLDB test: resolve SLF4J/Log4j version conflicts The HSQLDB test failed with NoClassDefFoundError on FrameworkLogger due to incompatible logging dependencies from Hive: - log4j-slf4j-impl (SLF4J 1.x bridge) conflicting with slf4j-api 2.0.6 - log4j-api 2.10.0 and log4j-core 2.18.0 version mismatch Fix by excluding log4j-slf4j-impl from all Hive dependencies, aligning Log4j2 at 2.24.3, and adding log4j-slf4j2-impl (SLF4J 2.x compatible). Also upgrade HSQLDB from 2.7.1 to 2.7.4. Co-Authored-By: Claude Sonnet 4.6 --- pom.xml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0fe88c17d..7c9a1106b 100644 --- a/pom.xml +++ b/pom.xml @@ -360,7 +360,7 @@ org.hsqldb hsqldb - 2.7.1 + 2.7.4 runtime @@ -382,16 +382,49 @@ org.apache.hive hive-jdbc 3.1.2 + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.hive hive-serde 4.0.1 + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.hive hive-cli 4.0.1 + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + org.apache.logging.log4j + log4j-api + 2.24.3 + + + org.apache.logging.log4j + log4j-core + 2.24.3 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.24.3 org.apache.hadoop From f928e7c98038097fcbcbde1cff85cf5d3549240a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 14:41:36 +0800 Subject: [PATCH 031/132] Fix CI job names: typos, copy-paste errors, and creation-only labels - CnosDB: fix step name "Set up ClickHouse" -> "Set up CnosDB" - CnosDB: add "creation only" (both NoREC and TLP use --num-queries 0) - HSQLDB: fix typo "HSQLB" -> "HSQLDB" - MySQL: note "CERT creation only" (CERT uses --num-queries 0) - TiDB: note "TLP creation only" (TLP uses --num-queries 0) Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f2e493f9..ed6769c34 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,7 +89,7 @@ jobs: run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test cnosdb: - name: DBMS Tests (CnosDB) + name: DBMS Tests (CnosDB, creation only) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -101,7 +101,7 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up ClickHouse + - name: Set up CnosDB run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest @@ -366,7 +366,7 @@ jobs: run: SPARK_AVAILABLE=true mvn -Dtest=TestSparkTLP test hsqldb: - name: DBMS Tests (HSQLB) + name: DBMS Tests (HSQLDB) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -458,7 +458,7 @@ jobs: MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQueryPlan mysql: - name: DBMS Tests (MySQL) + name: DBMS Tests (MySQL, CERT creation only) runs-on: ubuntu-latest services: mysql: @@ -601,7 +601,7 @@ jobs: mvn -Dtest=TestSQLiteQPG test tidb: - name: DBMS Tests (TiDB) + name: DBMS Tests (TiDB, TLP creation only) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 6a352e88c489f145c617c58f64eb1f4180b9a717 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 23:45:44 +0800 Subject: [PATCH 032/132] Remove unnecessary "does not exist" expected error from PostgresReindexGenerator Following the REINDEX syntax fix in PR #1283 (89a48dd2), the "does not exist" expected error is no longer needed. It was only masking the previous bug where multiple index names were concatenated without delimiters (e.g., REINDEX INDEX i0i1i2), causing PostgreSQL to report that the concatenated name does not exist. With the fix now selecting a single valid index, this error cannot legitimately occur since getIndexes() only returns indexes that exist in the schema. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresReindexGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index dc0d2cf34..d22ffe53e 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -58,7 +58,6 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { throw new AssertionError(scope); } errors.add("already contains data"); // FIXME bug report - errors.add("does not exist"); // internal index errors.add("REINDEX is not yet implemented for partitioned indexes"); return new SQLQueryAdapter(sb.toString(), errors); } From ce741c9eaa9f28130af5479a1e95f979a097d391 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 6 Apr 2026 09:53:54 +0800 Subject: [PATCH 033/132] Add EXPLAIN with randomized options for Postgres Based on PR #1265, which adds GENERIC_PLAN support for EXPLAIN. This reimplements the feature with the following bugs fixed: - EXPLAIN options were each wrapped in separate parentheses (e.g. EXPLAIN (ANALYZE) (FORMAT JSON) ...) instead of a single comma-separated list (EXPLAIN (ANALYZE, FORMAT JSON) ...) - BUFFERS and TIMING were generated without ANALYZE, which is required - GENERIC_PLAN and ANALYZE were not treated as mutually exclusive Closes #1044 point 3 Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/PostgresProvider.java | 4 ++ .../gen/PostgresExplainGenerator.java | 68 ++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index acd20a184..ec7978216 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -127,6 +127,7 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // + EXPLAIN(PostgresExplainGenerator::create), // CREATE_VIEW(PostgresViewGenerator::create), // CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); @@ -201,6 +202,9 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; + case EXPLAIN: + nrPerformed = r.getInteger(0, 1); + break; default: throw new AssertionError(a); } diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index d3039394b..e4359e5aa 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -1,16 +1,82 @@ package sqlancer.postgres.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTables; +import sqlancer.postgres.ast.PostgresSelect; + public final class PostgresExplainGenerator { private PostgresExplainGenerator() { } - public static String explain(String selectStr) throws Exception { + public static String explain(String selectStr) { StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN (FORMAT JSON) "); sb.append(selectStr); return sb.toString(); } + public static String explainGeneral(String selectStr) { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + + List options = new ArrayList<>(); + boolean analyze = Randomly.getBoolean(); + boolean genericPlan = !analyze && Randomly.getBoolean(); + if (analyze) { + options.add("ANALYZE"); + } + if (genericPlan) { + options.add("GENERIC_PLAN"); + } + if (Randomly.getBoolean()) { + options.add("FORMAT " + Randomly.fromOptions("TEXT", "XML", "JSON", "YAML")); + } + if (Randomly.getBoolean()) { + options.add("VERBOSE"); + } + if (Randomly.getBoolean()) { + options.add("COSTS"); + } + if (analyze && Randomly.getBoolean()) { + options.add("BUFFERS"); + } + if (analyze && Randomly.getBoolean()) { + options.add("TIMING"); + } + if (Randomly.getBoolean()) { + options.add("SUMMARY"); + } + if (!options.isEmpty()) { + sb.append("("); + sb.append(String.join(", ", options)); + sb.append(") "); + } + + sb.append(selectStr); + return sb.toString(); + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) throws Exception { + PostgresSchema.PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + gen.setTablesAndColumns(new PostgresTables(Arrays.asList(table))); + PostgresSelect select = gen.generateSelect(); + select.setFromList(gen.getTableRefs()); + select.setFetchColumns(gen.generateFetchColumns(false)); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PostgresDataType.BOOLEAN)); + } + return new SQLQueryAdapter(explainGeneral(select.asString())); + } + } From c67c7380ee39fa685895eea7839929aad41d44e4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Apr 2026 09:32:27 +0800 Subject: [PATCH 034/132] PostgreSQL: Add expected error for PRIMARY KEY on column with DROP NOT NULL When a multi-action ALTER TABLE combines ADD CONSTRAINT ... PRIMARY KEY with ALTER ... DROP NOT NULL, PostgreSQL returns "primary key column is not marked NOT NULL". Add this to expected errors in both ADD_TABLE_CONSTRAINT and ADD_TABLE_CONSTRAINT_USING_INDEX cases. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 5598c3f9f..69b509f60 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -304,6 +304,7 @@ public SQLQueryAdapter generate() { errors.add("multiple primary keys for table"); errors.add("could not create unique index"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("cannot cast type"); errors.add("unsupported PRIMARY KEY constraint with partition key definition"); errors.add("unsupported UNIQUE constraint with partition key definition"); @@ -342,6 +343,7 @@ public SQLQueryAdapter generate() { errors.add("appears twice in unique constraint"); errors.add("appears twice in primary key constraint"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("insufficient columns in PRIMARY KEY constraint definition"); errors.add("which is part of the partition key"); break; From 3db5ffa9c2153bcf5c36ea6278177e20063538b7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Apr 2026 09:34:42 +0800 Subject: [PATCH 035/132] PostgreSQL: Add expected error for PRIMARY KEY on column with DROP NOT NULL When a multi-action ALTER TABLE combines ADD CONSTRAINT ... PRIMARY KEY with ALTER ... DROP NOT NULL, PostgreSQL returns "primary key column is not marked NOT NULL". Add this to expected errors in both ADD_TABLE_CONSTRAINT and ADD_TABLE_CONSTRAINT_USING_INDEX cases. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 5598c3f9f..69b509f60 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -304,6 +304,7 @@ public SQLQueryAdapter generate() { errors.add("multiple primary keys for table"); errors.add("could not create unique index"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("cannot cast type"); errors.add("unsupported PRIMARY KEY constraint with partition key definition"); errors.add("unsupported UNIQUE constraint with partition key definition"); @@ -342,6 +343,7 @@ public SQLQueryAdapter generate() { errors.add("appears twice in unique constraint"); errors.add("appears twice in primary key constraint"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("insufficient columns in PRIMARY KEY constraint definition"); errors.add("which is part of the partition key"); break; From d8d90bbfefccd567270bda1de81b3dd0ff00f5ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 8 Apr 2026 00:35:01 +0800 Subject: [PATCH 036/132] Remove commented-out code in the DuckDB index generator --- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index cc4114f63..6c50b204d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -40,7 +40,6 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } } sb.append(")"); - // errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("cannot create an index on the rowid"); } From 6d8685090321a11648e2dd2f8538aadb02f937e4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 8 Apr 2026 09:55:17 +0800 Subject: [PATCH 037/132] Materialize: Update expected query plan for QPG test The latest Materialize version changed the EXPLAIN OPTIMIZED PLAN output format: the With block now appears before Return, and column name annotations shifted positions. Update the expected string to match. Co-Authored-By: Claude Opus 4.6 --- test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java index f282af42e..4d26bc08e 100644 --- a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java +++ b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java @@ -42,7 +42,7 @@ void testMaterializeQueryPlan() throws Exception { String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); assertEquals( - "Return // { arity: 3 };Union // { arity: 3 };Get l0 // { arity: 3 };Project (#2{c}, #3, #0) // { arity: 3 };Union // { arity: 1 };Negate // { arity: 1 };Project (#2) // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };With;ReadStorage queryplan.public.t1 // { arity: 2 };ReadStorage queryplan.public.t2 // { arity: 1 };;Source queryplan.public.t1;Source queryplan.public.t2;;Target cluster: quickstart;", + "With;ReadStorage queryplan.public.t1 // { arity: 2 };ReadStorage queryplan.public.t2 // { arity: 1 };Return // { arity: 3 };Union // { arity: 3 };Get l0 // { arity: 3 };Project (#2, #3, #0{c}) // { arity: 3 };Union // { arity: 1 };Negate // { arity: 1 };Project (#2{c}) // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };;Source queryplan.public.t1;Source queryplan.public.t2;;Target cluster: quickstart;", queryPlan); } From 9a9b5f345aa26571a815928ff96a24e41ffeee85 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Apr 2026 00:09:46 +0800 Subject: [PATCH 038/132] Use AbstractSchema.matchesViewName() for view detection everywhere Replace inlined tableName.startsWith("v") with the existing matchesViewName() helper across all 15 schema classes, ensuring a single source of truth for view name detection. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/clickhouse/ClickHouseSchema.java | 2 +- src/sqlancer/cockroachdb/CockroachDBSchema.java | 2 +- src/sqlancer/databend/DatabendSchema.java | 2 +- src/sqlancer/datafusion/DataFusionSchema.java | 2 +- src/sqlancer/doris/DorisSchema.java | 2 +- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- src/sqlancer/hive/HiveSchema.java | 2 +- src/sqlancer/hsqldb/HSQLDBSchema.java | 2 +- src/sqlancer/postgres/PostgresSchema.java | 2 +- src/sqlancer/presto/PrestoSchema.java | 2 +- src/sqlancer/questdb/QuestDBSchema.java | 2 +- src/sqlancer/spark/SparkSchema.java | 2 +- src/sqlancer/tidb/TiDBSchema.java | 2 +- src/sqlancer/yugabyte/ycql/YCQLSchema.java | 2 +- src/sqlancer/yugabyte/ysql/YSQLSchema.java | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index 97f4534a1..8f8f906ec 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -216,7 +216,7 @@ public static ClickHouseSchema fromConnection(SQLConnection con, String database for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); List indexes = Collections.emptyList(); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); ClickHouseTable t = new ClickHouseTable(tableName, databaseColumns, indexes, isView); for (ClickHouseColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 708afef92..c0eeedd2b 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -292,7 +292,7 @@ public static CockroachDBSchema fromConnection(SQLConnection con, String databas for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); List indexes = getIndexes(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); CockroachDBTable t = new CockroachDBTable(tableName, databaseColumns, indexes, isView); for (CockroachDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index af9a3e0ce..f6bf30757 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -321,7 +321,7 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa List tableNames = getTableNames(con, databaseName); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName, databaseName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); for (DatabendColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/datafusion/DataFusionSchema.java b/src/sqlancer/datafusion/DataFusionSchema.java index b9f00a02e..d02e80c30 100644 --- a/src/sqlancer/datafusion/DataFusionSchema.java +++ b/src/sqlancer/datafusion/DataFusionSchema.java @@ -37,7 +37,7 @@ public static DataFusionSchema fromConnection(SQLConnection con, String database for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); DataFusionTable t = new DataFusionTable(tableName, databaseColumns, isView); for (DataFusionColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/doris/DorisSchema.java b/src/sqlancer/doris/DorisSchema.java index 76697fad7..70a61ee62 100644 --- a/src/sqlancer/doris/DorisSchema.java +++ b/src/sqlancer/doris/DorisSchema.java @@ -569,7 +569,7 @@ public static DorisSchema fromConnection(SQLConnection con, String databaseName) continue; } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); DorisTable t = new DorisTable(tableName, databaseColumns, isView); for (DorisColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 8141220fa..e4b760221 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -231,7 +231,7 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName continue; // TODO: unexpected? } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); List indexes = getIndexes(con, tableName); DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java index 822eea163..8733d5caa 100644 --- a/src/sqlancer/hive/HiveSchema.java +++ b/src/sqlancer/hive/HiveSchema.java @@ -59,7 +59,7 @@ public static HiveSchema fromConnection(SQLConnection con, String databaseName) List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); HiveTable t = new HiveTable(tableName, databaseColumns, isView); for (HiveColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/hsqldb/HSQLDBSchema.java b/src/sqlancer/hsqldb/HSQLDBSchema.java index e1e1cb94b..2d41df83f 100644 --- a/src/sqlancer/hsqldb/HSQLDBSchema.java +++ b/src/sqlancer/hsqldb/HSQLDBSchema.java @@ -29,7 +29,7 @@ public static HSQLDBSchema fromConnection(SQLConnection connection, String datab continue; // TODO: unexpected? } List databaseColumns = getTableColumns(connection, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); HSQLDBSchema.HSQLDBTable t = new HSQLDBSchema.HSQLDBTable(tableName, databaseColumns, isView); for (HSQLDBSchema.HSQLDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 20337fea7..2c4607418 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -250,7 +250,7 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa boolean isPartitioned = "p".equals(rs.getString("relkind")); // TODO: also check insertable // TODO: insert into view? - boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || + boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || // tableTypeStr.contains("LOCAL TEMPORARY") && // !isInsertable; PostgresTable.TableType tableType = getTableType(tableTypeSchema); diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java index 112da7ef2..439615950 100644 --- a/src/sqlancer/presto/PrestoSchema.java +++ b/src/sqlancer/presto/PrestoSchema.java @@ -27,7 +27,7 @@ public static PrestoSchema fromConnection(SQLConnection con, String databaseName List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, databaseName, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); PrestoTable t = new PrestoTable(tableName, databaseColumns, isView); for (PrestoColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index 8253ec82c..55ee01aab 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -268,7 +268,7 @@ public static QuestDBSchema fromConnection(SQLConnection con, String databaseNam continue; // TODO: unexpected? } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); QuestDBTable t = new QuestDBTable(tableName, databaseColumns, isView); for (QuestDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java index 8d988e31e..9b3666916 100644 --- a/src/sqlancer/spark/SparkSchema.java +++ b/src/sqlancer/spark/SparkSchema.java @@ -54,7 +54,7 @@ public static SparkSchema fromConnection(SQLConnection con, String databaseName) List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.toLowerCase().startsWith("v"); + boolean isView = matchesViewName(tableName); SparkTable t = new SparkTable(tableName, databaseColumns, isView); for (SparkColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 32e00504b..f8439c734 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -313,7 +313,7 @@ public static TiDBSchema fromConnection(SQLConnection con, String databaseName) continue; } List indexes = getIndexes(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); TiDBTable t = new TiDBTable(tableName, databaseColumns, indexes, isView); for (TiDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java index 1534b9b56..41762d453 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLSchema.java +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -216,7 +216,7 @@ public static YCQLSchema fromConnection(SQLConnection con, String databaseName) continue; } List databaseColumns = getTableColumns(con, databaseName, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); YCQLTable t = new YCQLTable(tableName, databaseColumns, isView); for (YCQLColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/yugabyte/ysql/YSQLSchema.java b/src/sqlancer/yugabyte/ysql/YSQLSchema.java index 400a34e21..c75322af9 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLSchema.java +++ b/src/sqlancer/yugabyte/ysql/YSQLSchema.java @@ -81,7 +81,7 @@ public static YSQLSchema fromConnection(SQLConnection con, String databaseName) boolean isInsertable = rs.getBoolean("is_insertable_into"); // TODO: also check insertable // TODO: insert into view? - boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || + boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || // tableTypeStr.contains("LOCAL TEMPORARY") && // !isInsertable; YSQLTable.TableType tableType = getTableType(tableTypeSchema); From 5cb7193514b16a3668cb6b3bfdbb17ee8d8e384f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Apr 2026 00:11:55 +0800 Subject: [PATCH 039/132] Use getFreeViewName() in view generators instead of inline logic PostgresViewGenerator, MaterializeViewGenerator, and YSQLViewGenerator had their own inline "v" + i++ loops to find free view names. Replace with the existing AbstractSchema.getFreeViewName() which provides the same logic plus randomized start indices for better fuzzing variety. Co-Authored-By: Claude Opus 4.6 --- .../materialize/gen/MaterializeViewGenerator.java | 14 +++----------- .../postgres/gen/PostgresViewGenerator.java | 14 +++----------- .../yugabyte/ysql/gen/YSQLViewGenerator.java | 14 +++----------- 3 files changed, 9 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/materialize/gen/MaterializeViewGenerator.java b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java index c8cd0e93f..e3cb8ff29 100644 --- a/src/sqlancer/materialize/gen/MaterializeViewGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java @@ -30,19 +30,11 @@ public static SQLQueryAdapter create(MaterializeGlobalState globalState) { materialized = false; } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index b0a2a8b9d..10992ece6 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -35,19 +35,11 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { materialized = false; } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java index f70891cc2..3005d49f8 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -28,19 +28,11 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { } } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } From e6a46c5803e5021a6f34d35f9982719e09cc6178 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Apr 2026 09:12:41 +0800 Subject: [PATCH 040/132] Fix formatting in PostgresSchema after matchesViewName refactoring Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/PostgresSchema.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 2c4607418..8abd286f0 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -251,8 +251,8 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa // TODO: also check insertable // TODO: insert into view? boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || - // tableTypeStr.contains("LOCAL TEMPORARY") && - // !isInsertable; + // tableTypeStr.contains("LOCAL TEMPORARY") && + // !isInsertable; PostgresTable.TableType tableType = getTableType(tableTypeSchema); List databaseColumns = getTableColumns(con, tableName); List indexes = getIndexes(con, tableName); From 01d848de3ce1681b2fcc6d7393087e03bf606461 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Apr 2026 22:13:08 +0800 Subject: [PATCH 041/132] CI: Run reducer tests in Misc Tests job instead of separate job Combine the standalone reducer job into the existing Misc Tests step to reduce CI overhead. The reducer tests are lightweight and don't need their own runner. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed6769c34..d35bf55b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**,!sqlancer.reducer.**' test + run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - name: Set up Python uses: actions/setup-python@v4 with: @@ -711,21 +711,3 @@ jobs: DORIS_AVAILABLE=true mvn -Dtest=TestDorisNoREC test DORIS_AVAILABLE=true mvn -Dtest=TestDorisPQS test DORIS_AVAILABLE=true mvn -Dtest=TestDorisTLP test - - reducer: - name: Reducer Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - - name: Build - run: mvn -B package -DskipTests=true - - name: Run Tests - run: | - mvn -Dtest=TestStatementReducer test From 3cdb5c51ce006d3b72e324977c8c44cd51b46fa6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Apr 2026 22:18:43 +0800 Subject: [PATCH 042/132] Pull up isPrimaryKey() and hasPrimaryKey() into abstract base classes Add isPrimaryKey() to AbstractTableColumn (default false) and hasPrimaryKey() to AbstractTable, replacing duplicate implementations in MySQL, OceanBase, and TiDB. Add @Override to all database-specific isPrimaryKey() methods. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/cockroachdb/CockroachDBSchema.java | 1 + src/sqlancer/common/schema/AbstractTable.java | 4 ++++ src/sqlancer/common/schema/AbstractTableColumn.java | 4 ++++ src/sqlancer/databend/DatabendSchema.java | 1 + src/sqlancer/duckdb/DuckDBSchema.java | 1 + src/sqlancer/mariadb/MariaDBSchema.java | 1 + src/sqlancer/mysql/MySQLSchema.java | 5 +---- src/sqlancer/oceanbase/OceanBaseSchema.java | 5 +---- src/sqlancer/presto/PrestoSchema.java | 1 + src/sqlancer/sqlite3/schema/SQLite3Schema.java | 1 + src/sqlancer/tidb/TiDBSchema.java | 5 +---- src/sqlancer/yugabyte/ycql/YCQLSchema.java | 1 + 12 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index c0eeedd2b..cbde577cb 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -182,6 +182,7 @@ public CockroachDBColumn(String name, CockroachDBCompositeDataType columnType, b this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 89d568867..58154681c 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -95,6 +95,10 @@ public boolean isView() { return isView; } + public boolean hasPrimaryKey() { + return columns.stream().anyMatch(c -> c.isPrimaryKey()); + } + public String getFreeColumnName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/common/schema/AbstractTableColumn.java b/src/sqlancer/common/schema/AbstractTableColumn.java index e519bea97..a2f5fb1b3 100644 --- a/src/sqlancer/common/schema/AbstractTableColumn.java +++ b/src/sqlancer/common/schema/AbstractTableColumn.java @@ -12,6 +12,10 @@ public AbstractTableColumn(String name, T table, U type) { this.type = type; } + public boolean isPrimaryKey() { + return false; + } + public String getName() { return name; } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index f6bf30757..89738a1f3 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -148,6 +148,7 @@ public DatabendColumn(String name, DatabendCompositeDataType columnType, boolean this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index e4b760221..857b1e008 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -132,6 +132,7 @@ public DuckDBColumn(String name, DuckDBCompositeDataType columnType, boolean isP this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index 808653b25..7f7656d76 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -51,6 +51,7 @@ public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 0384f34df..c8a30614f 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -76,6 +76,7 @@ public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -194,10 +195,6 @@ public MySQLEngine getEngine() { return engine; } - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - } public static final class MySQLIndex extends TableIndex { diff --git a/src/sqlancer/oceanbase/OceanBaseSchema.java b/src/sqlancer/oceanbase/OceanBaseSchema.java index 7de2457ee..7b5b5954f 100644 --- a/src/sqlancer/oceanbase/OceanBaseSchema.java +++ b/src/sqlancer/oceanbase/OceanBaseSchema.java @@ -81,6 +81,7 @@ public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -195,10 +196,6 @@ public OceanBaseTable(String tableName, List columns, List c.isPrimaryKey()); - } - } public static final class OceanBaseIndex extends TableIndex { diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java index 439615950..2e668969d 100644 --- a/src/sqlancer/presto/PrestoSchema.java +++ b/src/sqlancer/presto/PrestoSchema.java @@ -453,6 +453,7 @@ public PrestoColumn(String name, PrestoCompositeDataType columnType, boolean isP this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 586d7e776..fc97929d3 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -78,6 +78,7 @@ public SQLite3Column(String rowId, SQLite3DataType columnType, boolean isInteger this.generated = generated; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index f8439c734..4ce7306e5 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -170,6 +170,7 @@ public TiDBColumn(String name, TiDBCompositeDataType columnType, boolean isPrima this.hasDefault = hasDefault; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -297,10 +298,6 @@ public TiDBTable(String tableName, List columns, List in super(tableName, columns, indexes, isView); } - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - } public static TiDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java index 41762d453..736247364 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLSchema.java +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -126,6 +126,7 @@ public YCQLColumn(String name, YCQLCompositeDataType columnType, boolean isPrima this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } From d13a1e4fc10956b14d6a580ee4aa99e876a6cd5c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Apr 2026 22:44:09 +0800 Subject: [PATCH 043/132] PostgreSQL: expect to_char "EEEE must be the last pattern used" error Random format strings fed to to_char (e.g. via md5(...)) can contain "eeee" followed by other characters, which PostgreSQL rejects with ERROR: "EEEE" must be the last pattern used. Add this to the known to_char expected errors so NoREC and similar oracles do not trip on it. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index e9f27aeb2..eeb160a56 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -122,6 +122,7 @@ private static List getToCharFunctionErrors() { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + errors.add("\"EEEE\" must be the last pattern used"); return errors; } From c9bc432f20d999edd73a9e9c3fe393c22a6ff2ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Apr 2026 23:40:04 +0800 Subject: [PATCH 044/132] CI: upgrade Citus from PostgreSQL 15 + Citus 11.1 to PostgreSQL 17 + Citus 13.0 ubuntu-latest is now Ubuntu 24.04 (Noble), which has no Citus packages. Patch citusdata_community.list to use jammy after the curl installer creates it, then re-run apt-get update before installing. Fixes: https://github.com/citusdata/citus/issues/7692 Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d35bf55b0..45055c819 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,9 +59,11 @@ jobs: run: | echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list curl https://install.citusdata.com/community/deb.sh | sudo bash - sudo apt-get -y install postgresql-15-citus-11.1 + sudo sed -i 's/noble/jammy/g' /etc/apt/sources.list.d/citusdata_community.list # https://github.com/citusdata/citus/issues/7692 + sudo apt-get update + sudo apt-get -y install postgresql-17-citus-13.0 sudo chown -R $USER:$USER /var/run/postgresql - export PATH=/usr/lib/postgresql/15/bin:$PATH + export PATH=/usr/lib/postgresql/17/bin:$PATH cd ~ mkdir -p citus/coordinator citus/worker1 citus/worker2 initdb -D citus/coordinator From 9780e5f58715421f989e0885966da30b08838df5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Apr 2026 00:24:20 +0800 Subject: [PATCH 045/132] Fix Citus CI: handle reg* OID types in PostgresSchema PostgreSQL 17 + Citus 13.0 system tables expose columns with OID alias types (regnamespace, regrole, regtype, regproc, etc.) that were not handled by getColumnType(), causing an AssertionError. Map them to TEXT since they are textually representable, like the existing regclass entry. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/PostgresSchema.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 8abd286f0..82937557c 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -125,6 +125,12 @@ public static PostgresDataType getColumnType(String typeString) { case "character varying": case "name": case "regclass": + case "regnamespace": + case "regrole": + case "regtype": + case "regproc": + case "regprocedure": + case "regoper": return PostgresDataType.TEXT; case "numeric": return PostgresDataType.DECIMAL; From a2019cc2f2358cbeb769024ffe081d5b98f99e38 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Apr 2026 00:25:27 +0800 Subject: [PATCH 046/132] Fix Citus CI: filter citus_* system views and update error strings for Citus 13.0 - CitusSchema.fromConnection now skips all tables/views starting with "citus_" (not just "citus_tables") to handle the citus_schemas view added in Citus 12.0, which was causing unexpected SQL errors when SQLancer tried to DELETE/ALTER it after the regnamespace fix exposed it. - Update columnar table index error string from the old (incorrect) "indexes not supported for columnar tables" to the actual Citus error "unsupported access method for the index on columnar table", and add "BRIN indexes on columnar tables are not supported". - Add "alter table command is currently unsupported" for the Citus restriction on certain ALTER TABLE operations (e.g., multi-subcommand ALTERs, type changes) on distributed tables. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/citus/CitusSchema.java | 4 ++-- src/sqlancer/citus/gen/CitusCommon.java | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index 560191c05..b2550bdce 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -63,8 +63,8 @@ public static CitusSchema fromConnection(SQLConnection con, String databaseName) "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { while (rs.next()) { String tableName = rs.getString("table_name"); - /* citus_tables is a helper view, we don't need to test with it so we let's ignore it */ - if (tableName.equals("citus_tables")) { + /* skip Citus-managed views in the public schema (citus_tables, citus_schemas, etc.) */ + if (tableName.startsWith("citus_")) { continue; } String distributionColumnName = rs.getString("dist_col_name"); diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 7a9f9c659..35aee85fa 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -23,10 +23,10 @@ public static List getCitusErrors() { errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); errors.add("functions used in UPDATE queries on distributed tables must not be VOLATILE"); errors.add("STABLE functions used in UPDATE queries cannot be called with column references"); - errors.add( - "functions used in the WHERE clause of modification queries on distributed tables must not be VOLATILE"); + errors.add("of modification queries on distributed tables must not be VOLATILE"); errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); errors.add("cannot execute ALTER TABLE command involving partition column"); + errors.add("alter table command is currently unsupported"); errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); errors.add("is not a regular, foreign or partitioned table"); errors.add("must be a distributed table or a reference table"); @@ -53,7 +53,8 @@ public static List getCitusErrors() { errors.add("direct joins between distributed and local tables are not supported"); errors.add("unlogged columnar tables are not supported"); errors.add("UPDATE and CTID scans not supported for ColumnarScan"); - errors.add("indexes not supported for columnar tables"); + errors.add("unsupported access method for the index on columnar table"); + errors.add("BRIN indexes on columnar tables are not supported"); errors.add("invalid byte sequence for encoding \"UTF8\": 0x00"); errors.add("columnar_tuple_insert_speculative not implemented"); errors.add("row field count is 1, expected 2"); From aed787157599f706c4d0d705d01b1b2fa7299efa Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Apr 2026 00:25:51 +0800 Subject: [PATCH 047/132] Fix Citus CI: handle OID 0 errors for columnar temporary tables Citus 13.0 raises "could not open relation with OID 0" when operating on columnar temporary tables (USING columnar ON COMMIT DROP) during VACUUM, DISCARD TEMPORARY, and some INSERT operations where Citus cannot resolve the relation OID. Collect the error into getColumnarOidErrors() and wire it through Citus-specific generator wrappers: - CitusVacuumGenerator wraps PostgresVacuumGenerator so VACUUM accepts the OID 0 error (the VACUUM action previously used the Postgres generator directly). - CitusDiscardGenerator does the same for DISCARD. - INSERT already routes through CitusInsertGenerator, which picks up the error via addCitusErrors() in getCitusErrors(). Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/citus/CitusProvider.java | 16 +++++++-------- src/sqlancer/citus/gen/CitusCommon.java | 15 ++++++++++++++ .../citus/gen/CitusDiscardGenerator.java | 20 +++++++++++++++++++ .../citus/gen/CitusReindexGenerator.java | 20 +++++++++++++++++++ .../citus/gen/CitusTruncateGenerator.java | 20 +++++++++++++++++++ .../citus/gen/CitusVacuumGenerator.java | 20 +++++++++++++++++++ 6 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/citus/gen/CitusDiscardGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusReindexGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusTruncateGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusVacuumGenerator.java diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 747f72216..a11424f18 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -21,11 +21,15 @@ import sqlancer.citus.gen.CitusAlterTableGenerator; import sqlancer.citus.gen.CitusCommon; import sqlancer.citus.gen.CitusDeleteGenerator; +import sqlancer.citus.gen.CitusDiscardGenerator; import sqlancer.citus.gen.CitusIndexGenerator; import sqlancer.citus.gen.CitusInsertGenerator; +import sqlancer.citus.gen.CitusReindexGenerator; import sqlancer.citus.gen.CitusSetGenerator; import sqlancer.citus.gen.CitusTableGenerator; +import sqlancer.citus.gen.CitusTruncateGenerator; import sqlancer.citus.gen.CitusUpdateGenerator; +import sqlancer.citus.gen.CitusVacuumGenerator; import sqlancer.citus.gen.CitusViewGenerator; import sqlancer.common.DBMSCommon; import sqlancer.common.oracle.CompositeTestOracle; @@ -44,15 +48,11 @@ import sqlancer.postgres.gen.PostgresAnalyzeGenerator; import sqlancer.postgres.gen.PostgresClusterGenerator; import sqlancer.postgres.gen.PostgresCommentGenerator; -import sqlancer.postgres.gen.PostgresDiscardGenerator; import sqlancer.postgres.gen.PostgresDropIndexGenerator; import sqlancer.postgres.gen.PostgresNotifyGenerator; -import sqlancer.postgres.gen.PostgresReindexGenerator; import sqlancer.postgres.gen.PostgresSequenceGenerator; import sqlancer.postgres.gen.PostgresStatisticsGenerator; import sqlancer.postgres.gen.PostgresTransactionGenerator; -import sqlancer.postgres.gen.PostgresTruncateGenerator; -import sqlancer.postgres.gen.PostgresVacuumGenerator; @AutoService(DatabaseProvider.class) public class CitusProvider extends PostgresProvider { @@ -82,13 +82,13 @@ public enum Action implements AbstractAction { CREATE_STATISTICS(PostgresStatisticsGenerator::insert), // DROP_STATISTICS(PostgresStatisticsGenerator::remove), // DELETE(CitusDeleteGenerator::create), // - DISCARD(PostgresDiscardGenerator::create), // + DISCARD(CitusDiscardGenerator::create), // DROP_INDEX(PostgresDropIndexGenerator::create), // INSERT(CitusInsertGenerator::insert), // UPDATE(CitusUpdateGenerator::create), // - TRUNCATE(PostgresTruncateGenerator::create), // - VACUUM(PostgresVacuumGenerator::create), // - REINDEX(PostgresReindexGenerator::create), // + TRUNCATE(CitusTruncateGenerator::create), // + VACUUM(CitusVacuumGenerator::create), // + REINDEX(CitusReindexGenerator::create), // SET(CitusSetGenerator::create), // CREATE_INDEX(CitusIndexGenerator::generate), // SET_CONSTRAINTS((g) -> { diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 35aee85fa..ba8936aaa 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -27,6 +27,7 @@ public static List getCitusErrors() { errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); errors.add("cannot execute ALTER TABLE command involving partition column"); errors.add("alter table command is currently unsupported"); + errors.add("on distributed partitioned tables are not supported"); errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); errors.add("is not a regular, foreign or partitioned table"); errors.add("must be a distributed table or a reference table"); @@ -61,6 +62,7 @@ public static List getCitusErrors() { errors.add("incorrect binary data format"); errors.add("invalid sign in external \"numeric\" value"); errors.add("Foreign keys and AFTER ROW triggers are not supported for columnar tables"); + errors.addAll(getColumnarOidErrors()); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { @@ -88,6 +90,19 @@ public static List getCitusErrors() { return errors; } + /** + * Citus can fail with "could not open relation with OID 0" when operating on columnar temporary tables (e.g., USING + * columnar ON COMMIT DROP), during VACUUM, DISCARD TEMPORARY, or INSERT operations where Citus cannot resolve the + * relation OID. + * + * @return the list of expected error substrings for columnar OID resolution failures. + */ + public static List getColumnarOidErrors() { + List errors = new ArrayList<>(); + errors.add("could not open relation with OID 0"); + return errors; + } + public static void addCitusErrors(ExpectedErrors errors) { errors.addAll(getCitusErrors()); } diff --git a/src/sqlancer/citus/gen/CitusDiscardGenerator.java b/src/sqlancer/citus/gen/CitusDiscardGenerator.java new file mode 100644 index 000000000..f4a1b3240 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusDiscardGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresDiscardGenerator; + +public final class CitusDiscardGenerator { + + private CitusDiscardGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter discardQuery = PostgresDiscardGenerator.create(globalState); + ExpectedErrors errors = discardQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return discardQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusReindexGenerator.java b/src/sqlancer/citus/gen/CitusReindexGenerator.java new file mode 100644 index 000000000..6f37cbe06 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusReindexGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresReindexGenerator; + +public final class CitusReindexGenerator { + + private CitusReindexGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter reindexQuery = PostgresReindexGenerator.create(globalState); + ExpectedErrors errors = reindexQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return reindexQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusTruncateGenerator.java b/src/sqlancer/citus/gen/CitusTruncateGenerator.java new file mode 100644 index 000000000..cf36ce9c2 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusTruncateGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresTruncateGenerator; + +public final class CitusTruncateGenerator { + + private CitusTruncateGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter truncateQuery = PostgresTruncateGenerator.create(globalState); + ExpectedErrors errors = truncateQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return truncateQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusVacuumGenerator.java b/src/sqlancer/citus/gen/CitusVacuumGenerator.java new file mode 100644 index 000000000..ae73dbf82 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusVacuumGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresVacuumGenerator; + +public final class CitusVacuumGenerator { + + private CitusVacuumGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter vacuumQuery = PostgresVacuumGenerator.create(globalState); + ExpectedErrors errors = vacuumQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return vacuumQuery; + } + +} From f88317414049f709632a88efe442d660bc7f7002 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 15 Apr 2026 09:49:49 +0800 Subject: [PATCH 048/132] CI: skip JaCoCo report for test runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JaCoCo report goal intermittently fails with EOFException when the forked test JVM is force-killed before the agent can finish flushing jacoco.exec. This has been especially visible on QPG/Materialize and other remote-DBMS jobs where long-lived worker threads continue past test completion. The report isn't uploaded or consumed anywhere in CI — it's generated and discarded — so skip it with -Djacoco.skip=true. Developers can still produce coverage locally by running mvn test without the flag. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 108 ++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45055c819..40306d847 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test + run: mvn -Djacoco.skip=true -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - name: Set up Python uses: actions/setup-python@v4 with: @@ -88,7 +88,7 @@ jobs: psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test - name: Run Tests - run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test + run: CITUS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCitus test cnosdb: name: DBMS Tests (CnosDB, creation only) @@ -110,9 +110,9 @@ jobs: until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done - name: Run Tests run: | - CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test + CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBNoREC test sleep 20 - CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBTLP test + CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBTLP test clickhouse: name: DBMS Tests (ClickHouse) @@ -133,7 +133,7 @@ jobs: docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:24.3.1.2672 until curl -sf http://127.0.0.1:8123/ping 2>/dev/null; do sleep 1; done - name: Run Tests - run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test + run: CLICKHOUSE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test - name: Show fatal errors run: docker exec clickhouse-server grep Fatal /var/log/clickhouse-server/clickhouse-server.log || echo No Fatal Errors found - name: Teardown ClickHouse server @@ -163,9 +163,9 @@ jobs: run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: | - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBNoREC test - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBTLP test - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBCERT test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBNoREC test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBTLP test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBCERT test cockroachdb-qpg: name: QPG Tests (CockroachDB) @@ -188,7 +188,7 @@ jobs: - name: Create SQLancer user run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests - run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test + run: COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBQPG test databend: name: DBMS Tests (Databend) @@ -214,9 +214,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendTLP test - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendNoREC test - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendPQS test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendTLP test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendNoREC test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendPQS test datafusion: name: DBMS Tests (DataFusion) @@ -258,7 +258,7 @@ jobs: exit 1 - name: Run Tests run: | - DATAFUSION_AVAILABLE=true mvn test -Pdatafusion-tests + DATAFUSION_AVAILABLE=true mvn -Djacoco.skip=true test -Pdatafusion-tests duckdb: name: DBMS Tests (DuckDB) @@ -276,8 +276,8 @@ jobs: run: mvn -B package -DskipTests=true - name: DuckDB Tests run: | - mvn -Dtest=TestDuckDBTLP test - mvn -Dtest=TestDuckDBNoREC test + mvn -Djacoco.skip=true -Dtest=TestDuckDBTLP test + mvn -Djacoco.skip=true -Dtest=TestDuckDBNoREC test h2: name: DBMS Tests (H2) @@ -293,7 +293,7 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: mvn -Dtest=TestH2 test + run: mvn -Djacoco.skip=true -Dtest=TestH2 test hive: name: DBMS Tests (Hive) @@ -327,7 +327,7 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: HIVE_AVAILABLE=true mvn -Dtest=TestHiveTLP test + run: HIVE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestHiveTLP test spark: name: DBMS Tests (Spark) @@ -365,7 +365,7 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests - run: SPARK_AVAILABLE=true mvn -Dtest=TestSparkTLP test + run: SPARK_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestSparkTLP test hsqldb: name: DBMS Tests (HSQLDB) @@ -382,8 +382,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - mvn -Dtest=TestHSQLDBNoREC test - mvn -Dtest=TestHSQLDBTLP test + mvn -Djacoco.skip=true -Dtest=TestHSQLDBNoREC test + mvn -Djacoco.skip=true -Dtest=TestHSQLDBTLP test mariadb: name: DBMS Tests (MariaDB) @@ -410,7 +410,7 @@ jobs: - name: Create SQLancer User run: sudo mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests - run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test + run: MARIADB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestMariaDB test materialize: name: DBMS Tests (Materialize) @@ -432,9 +432,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeNoREC - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeTLP - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializePQS + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeNoREC + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeTLP + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializePQS materialize-qpg: name: QPG Tests (Materialize) @@ -456,8 +456,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQPG - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQueryPlan + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeQPG + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeQueryPlan mysql: name: DBMS Tests (MySQL, CERT creation only) @@ -484,10 +484,10 @@ jobs: run: mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLCERT - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLDQE + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLPQS + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLTLP + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLCERT + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLDQE oceanbase: name: DBMS Tests (OceanBase) @@ -509,9 +509,9 @@ jobs: mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | - OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseNoREC - OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBasePQS - OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseTLP + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBaseNoREC + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBasePQS + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBaseTLP postgres: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest @@ -534,10 +534,10 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresPQS test - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresTLP test - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresNoREC test - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresCERT test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresPQS test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresTLP test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresNoREC test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresCERT test presto: name: DBMS Tests (Presto) @@ -560,9 +560,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoNoREC test + PRESTO_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPrestoNoREC test docker restart presto && until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done - PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoTLP test + PRESTO_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPrestoTLP test sqlite: name: DBMS Tests (SQLite) runs-on: ubuntu-latest @@ -579,10 +579,10 @@ jobs: run: mvn -B package -DskipTests=true - name: SQLite Tests run: | - mvn -Dtest=TestSQLitePQS test - mvn -Dtest=TestSQLiteTLP test - mvn -Dtest=TestSQLiteNoREC test - mvn -Dtest=TestSQLiteCODDTest test + mvn -Djacoco.skip=true -Dtest=TestSQLitePQS test + mvn -Djacoco.skip=true -Dtest=TestSQLiteTLP test + mvn -Djacoco.skip=true -Dtest=TestSQLiteNoREC test + mvn -Djacoco.skip=true -Dtest=TestSQLiteCODDTest test sqlite-qpg: name: QPG Tests (SQLite) @@ -600,7 +600,7 @@ jobs: run: mvn -B package -DskipTests=true - name: SQLite Tests for QPG run: | - mvn -Dtest=TestSQLiteQPG test + mvn -Djacoco.skip=true -Dtest=TestSQLiteQPG test tidb: name: DBMS Tests (TiDB, TLP creation only) @@ -624,8 +624,8 @@ jobs: run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: | - TIDB_AVAILABLE=true mvn -Dtest=TestTiDBTLP test - TIDB_AVAILABLE=true mvn -Dtest=TestTiDBCERT test + TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBTLP test + TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBCERT test tidb-qpg: name: QPG Tests (TiDB) @@ -648,7 +648,7 @@ jobs: - name: Create SQLancer user run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests - run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test + run: TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBQPG test yugabyte: name: DBMS Tests (YugabyteDB) @@ -670,10 +670,10 @@ jobs: until pg_isready -h localhost -p 5433; do sleep 1; done - name: Run Tests run: | - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLNoREC test - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLTLP test - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLPQS test - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYCQL test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLNoREC test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLTLP test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLPQS test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYCQL test doris: name: DBMS Tests (Apache Doris) @@ -710,6 +710,6 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - DORIS_AVAILABLE=true mvn -Dtest=TestDorisNoREC test - DORIS_AVAILABLE=true mvn -Dtest=TestDorisPQS test - DORIS_AVAILABLE=true mvn -Dtest=TestDorisTLP test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisNoREC test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisPQS test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisTLP test From 8804b3f1d627426a64db40ca5d221e04d91872b3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 16 Apr 2026 22:38:19 +0800 Subject: [PATCH 049/132] Fix YugabyteDB CI: use DROP DATABASE WITH (FORCE) DROP DATABASE could fail with "database is being accessed by other users" when a prior iteration's session had not yet been released, causing tests to exit with -1. Adding WITH (FORCE) terminates lingering sessions as part of the drop. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/yugabyte/ysql/YSQLProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java index e72790344..efcc7ec22 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLProvider.java +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -204,11 +204,11 @@ private void createDatabaseSync(YSQLGlobalState globalState, String entryDatabas Connection con = createConnectionSafely(entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)"); createDatabaseCommand = getCreateDatabaseCommand(globalState); globalState.getState().logStatement(createDatabaseCommand); try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + s.execute("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)"); } try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); From 0c477bbfb553ce4e158f4f49b516e990f7b10b68 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 16 Apr 2026 23:16:43 +0800 Subject: [PATCH 050/132] Fix Citus CI: add distribution-column filter error for recursive CTEs as expected error Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/citus/gen/CitusCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index ba8936aaa..58b1b7c16 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -18,6 +18,7 @@ public static List getCitusErrors() { errors.add("cannot perform an INSERT without a partition column value"); errors.add("cannot perform an INSERT with NULL in the partition column"); errors.add("recursive CTEs are not supported in distributed queries"); + errors.add("recursive CTEs are only supported when they contain a filter on the distribution column"); errors.add("could not run distributed query with GROUPING SETS, CUBE, or ROLLUP"); errors.add("Subqueries in HAVING cannot refer to outer query"); errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); From e44eba43304e7ea0d4f65c639fa9a13f06bfc5d9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 17 Apr 2026 23:06:03 +0800 Subject: [PATCH 051/132] Fix Databend CI: upgrade to v1.2.896-nightly and handle bug #19738 The old Databend image (v1.2.687-nightly) had a server hang triggered by SELECT DISTINCT with large LIMIT values, which caused PQS tests to fail on every CI run. Upgrade to v1.2.896-nightly (which fixes the hang) and work around https://github.com/databendlabs/databend/issues/19738, where SELECT AVG(constant) over a cross join causes an internal assertion failure (UnwindError with Decimal precision mismatch). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/DatabendBugs.java | 1 + src/sqlancer/databend/DatabendErrors.java | 4 ++++ .../tlp/DatabendQueryPartitioningAggregateTester.java | 11 ++++++++--- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 40306d847..ee4186650 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.687-nightly + image: datafuselabs/databend:v1.2.896-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index dd11512d8..a058ccc32 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -19,6 +19,7 @@ public final class DatabendBugs { public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 + public static boolean bug19738 = true; // https://github.com/databendlabs/databend/issues/19738 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index fdd8a3a69..746a4e848 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -47,6 +47,10 @@ public static List getExpressionErrors() { if (DatabendBugs.bug15568) { errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); } + if (DatabendBugs.bug19738) { + errors.add("UnwindError"); + errors.add("unable to cast `NULL`"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 0ba0d9c86..6d52caea3 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -10,6 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -44,9 +45,13 @@ public DatabendQueryPartitioningAggregateTester(DatabendGlobalState state) { @Override public void check() throws SQLException { super.check(); - DatabendAggregateFunction aggregateFunction = Randomly.fromOptions(DatabendAggregateFunction.MAX, - DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT, - DatabendAggregateFunction.AVG/* , DatabendAggregateFunction.STDDEV_POP */); + List aggregateFunctions = new ArrayList<>(List.of(DatabendAggregateFunction.MAX, + DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT + /* , DatabendAggregateFunction.STDDEV_POP */)); + if (!DatabendBugs.bug19738) { + aggregateFunctions.add(DatabendAggregateFunction.AVG); + } + DatabendAggregateFunction aggregateFunction = Randomly.fromList(aggregateFunctions); DatabendFunctionOperation aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); List fetchColumns = new ArrayList<>(); From d6533d2e4c4453ce8947f3f9420862e5892e0750 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 00:07:58 +0800 Subject: [PATCH 052/132] Fix Presto CI: handle bytecode compiler bugs #27608 and #27609 Walk the exception cause chain in ComparatorHelper (consistent with SQLQueryAdapter.checkException) so wrapped errors like the JDBC driver's RuntimeException are matched against expected errors. Add VerifyError (#27608) and Compiler failed (#27609) as expected Presto bugs triggered by complex CASE expressions on Presto 0.297. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/ComparatorHelper.java | 11 ++++++----- src/sqlancer/presto/PrestoBugs.java | 6 ++++++ src/sqlancer/presto/PrestoErrors.java | 7 +++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 5da635de2..cee290924 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -70,11 +70,12 @@ public static List getResultSetFirstColumnAsString(String queryString, E throw e; } - if (e.getMessage() == null) { - throw new AssertionError(queryString, e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); + Throwable current = e; + while (current != null) { + if (current.getMessage() != null && errors.errorIsExpected(current.getMessage())) { + throw new IgnoreMeException(); + } + current = current.getCause(); } throw new AssertionError(queryString, e); } finally { diff --git a/src/sqlancer/presto/PrestoBugs.java b/src/sqlancer/presto/PrestoBugs.java index f5e888df1..b0eb3fe57 100644 --- a/src/sqlancer/presto/PrestoBugs.java +++ b/src/sqlancer/presto/PrestoBugs.java @@ -8,6 +8,12 @@ public final class PrestoBugs { // https://github.com/prestodb/presto/issues/23613 public static boolean bug23613 = true; + // https://github.com/prestodb/presto/issues/27608 + public static boolean bugVerifyError = true; + + // https://github.com/prestodb/presto/issues/27609 + public static boolean bugCompilerFailed = true; + private PrestoBugs() { } diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 2296223d6..dd2931976 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -47,6 +47,13 @@ public static List getExpressionErrors() { } errors.add("Cannot cast java.lang.String to java.util.List"); errors.add("Unexpected subquery expression in logical plan"); + if (PrestoBugs.bugVerifyError) { + errors.add("VerifyError"); + } + if (PrestoBugs.bugCompilerFailed) { + errors.add("Compiler failed"); + errors.add("Error processing class definition"); + } // 9223372036854775808 errors.add("Invalid numeric literal"); From e5ea4affea8271210966c04b89367e1f3d506b30 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 00:23:07 +0800 Subject: [PATCH 053/132] Fix Hive CI: skip unary prefix operators to work around negation nullability bug Hive incorrectly evaluates IS NULL for negated expressions involving string concatenation with column references (e.g., -(c || 'x') IS NULL returns false instead of true). The optimizer's nullability inference for GenericUDFOPNegative does not account for runtime NULL from non-null input. Affects Hive 4.0.1 and 4.2.0. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/hive/HiveBugs.java | 18 ++++++++++++++++++ .../hive/gen/HiveExpressionGenerator.java | 9 ++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/hive/HiveBugs.java diff --git a/src/sqlancer/hive/HiveBugs.java b/src/sqlancer/hive/HiveBugs.java new file mode 100644 index 000000000..01b772bb8 --- /dev/null +++ b/src/sqlancer/hive/HiveBugs.java @@ -0,0 +1,18 @@ +package sqlancer.hive; + +// do not make the fields final to avoid warnings +public final class HiveBugs { + + // Incorrect IS NULL evaluation for negation of string concatenation involving column references. + // -(c || 'x') evaluates to NULL at runtime, but IS NULL incorrectly returns false. + // The optimizer's nullability inference for GenericUDFOPNegative does not account for + // runtime conversion failures producing NULL from non-null input. + // Reproduce: CREATE TABLE t(c DOUBLE); INSERT INTO t VALUES(1.0); + // SELECT (-(c || 'x')) IS NULL FROM t; -- returns false, expected true + // Affects: 4.0.1, 4.2.0 + public static boolean bugNegationNullability = true; + + private HiveBugs() { + } + +} diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index 9f3de2514..a31420899 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -5,12 +5,14 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; +import sqlancer.hive.HiveBugs; import sqlancer.hive.HiveGlobalState; import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveDataType; @@ -80,7 +82,12 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr Expression expr = Randomly.fromList(possibleOptions); switch (expr) { case UNARY_PREFIX: - return new HiveUnaryPrefixOperation(generateExpression(depth + 1), HiveUnaryPrefixOperator.getRandom()); + HiveUnaryPrefixOperator prefixOp = HiveUnaryPrefixOperator.getRandom(); + if (HiveBugs.bugNegationNullability + && (prefixOp == HiveUnaryPrefixOperator.MINUS || prefixOp == HiveUnaryPrefixOperator.PLUS)) { + throw new IgnoreMeException(); + } + return new HiveUnaryPrefixOperation(generateExpression(depth + 1), prefixOp); case UNARY_POSTFIX: return new HiveUnaryPostfixOperation(generateExpression(depth + 1), HiveUnaryPostfixOperator.getRandom()); case BINARY_COMPARISON: From 8ee570f530b01890d77e7f101ec34ee47904bef7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 00:44:50 +0800 Subject: [PATCH 054/132] Fix OceanBase CI: add "value is out of range" as expected expression error Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/oceanbase/OceanBaseErrors.java | 1 + src/sqlancer/oceanbase/OceanBaseOracleFactory.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index c539ab129..7a5bd8a77 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -15,6 +15,7 @@ public static List getExpressionErrors() { ArrayList errors = new ArrayList<>(); errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) + errors.add("value is out of range"); errors.add("is not valid for CHARACTER SET"); errors.add("The observer or zone is not the master"); errors.add("Incorrect integer value"); diff --git a/src/sqlancer/oceanbase/OceanBaseOracleFactory.java b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java index b7b115f37..b1ab1cb5b 100644 --- a/src/sqlancer/oceanbase/OceanBaseOracleFactory.java +++ b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java @@ -17,7 +17,7 @@ public enum OceanBaseOracleFactory implements OracleFactory create(OceanBaseGlobalState globalState) throws SQLException { OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) - .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).with("value is out of range").build(); + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).build(); return new TLPWhereOracle<>(globalState, gen, expectedErrors); } From 220137f84371f2ebf64c1649f1f741d96c7fb056 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 01:12:55 +0800 Subject: [PATCH 055/132] Fix Hive CI: skip additional expression types triggering Hive evaluation bugs Add bug flags for three more Hive bugs: - bugNonBooleanWhereClause: CAST/FLOOR/ROUND/arithmetic silently return 0 rows when used as WHERE predicates instead of erroring - bugInBooleanEvaluation: IN with boolean/IS NULL sub-expressions returns 0 rows for all TLP partitions - bugBetweenMixedTypes: BETWEEN with mixed boolean/numeric types loses rows in TLP partitions Skip CAST, FUNC, BINARY_ARITHMETIC, IN, and BETWEEN expression types in the generator when the corresponding bug flags are enabled. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/hive/HiveBugs.java | 23 +++++++++++++++++++ .../hive/gen/HiveExpressionGenerator.java | 12 +++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/hive/HiveBugs.java b/src/sqlancer/hive/HiveBugs.java index 01b772bb8..43d42ce0b 100644 --- a/src/sqlancer/hive/HiveBugs.java +++ b/src/sqlancer/hive/HiveBugs.java @@ -12,6 +12,29 @@ public final class HiveBugs { // Affects: 4.0.1, 4.2.0 public static boolean bugNegationNullability = true; + // Non-boolean expressions (CAST to non-boolean, FLOOR, ROUND, arithmetic) silently + // return 0 rows for all three TLP partitions when used as WHERE predicates. + // Hive requires BOOLEAN in WHERE but does not error; instead it returns empty results. + // Reproduce: CREATE TABLE t(c INT); INSERT INTO t VALUES(1); + // SELECT * FROM t WHERE FLOOR(1); -- returns 0 rows, expected 1 + // Affects: 4.0.1, 4.2.0 + public static boolean bugNonBooleanWhereClause = true; + + // IN operator with boolean sub-expressions involving IS NULL evaluates incorrectly, + // returning 0 rows for all three TLP partitions. + // Reproduce: CREATE TABLE t(c BOOLEAN); INSERT INTO t VALUES(true),(false); + // SELECT * FROM t WHERE (c != c) IN ((false) IS NULL); -- returns 0, expected 2 + // Affects: 4.0.1, 4.2.0 + public static boolean bugInBooleanEvaluation = true; + + // BETWEEN with mixed boolean/numeric types has incorrect TLP evaluation. + // The IS NULL partition misses rows due to wrong nullability inference. + // Reproduce: CREATE TABLE t(c DOUBLE); INSERT INTO t VALUES(0.5),(1.5); + // SELECT * FROM t WHERE (c NOT IN (true)) NOT BETWEEN 0.01 AND c; + // -- TLP partitions lose rows + // Affects: 4.0.1, 4.2.0 + public static boolean bugBetweenMixedTypes = true; + private HiveBugs() { } diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index a31420899..92154873c 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -77,7 +77,17 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - // TODO: remove some of the possible expression types according to options. + if (HiveBugs.bugNonBooleanWhereClause) { + possibleOptions.remove(Expression.CAST); + possibleOptions.remove(Expression.FUNC); + possibleOptions.remove(Expression.BINARY_ARITHMETIC); + } + if (HiveBugs.bugInBooleanEvaluation) { + possibleOptions.remove(Expression.IN); + } + if (HiveBugs.bugBetweenMixedTypes) { + possibleOptions.remove(Expression.BETWEEN); + } Expression expr = Randomly.fromList(possibleOptions); switch (expr) { From e81ffd1ddbf326f485a79ed9dd28906263fd7cb1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 13:56:04 +0800 Subject: [PATCH 056/132] Fix Materialize CI: retry readSchema to handle eventual consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Materialize's information_schema is eventually consistent — tables and columns may not be visible immediately after creation. This caused IndexOutOfBoundsException when generators called getRandomTable() on an empty or incomplete schema. Retry readSchema() until the snapshot has no tables with empty columns, the table count has not regressed, and — after the initial read — the table list is not suspiciously empty. Co-Authored-By: Claude Opus 4.6 --- .../materialize/MaterializeGlobalState.java | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java index 77cbe5d14..8a66a7819 100644 --- a/src/sqlancer/materialize/MaterializeGlobalState.java +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -27,6 +27,8 @@ public class MaterializeGlobalState extends SQLGlobalState functionsAndTypes = new HashMap<>(); private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + private int lastKnownTableCount; + private int readSchemaCallCount; @Override public void setConnection(SQLConnection con) { @@ -266,7 +268,30 @@ public String getRandomTableAccessMethod() { @Override public MaterializeSchema readSchema() throws SQLException { - return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + // Materialize's information_schema is eventually consistent: tables and columns + // may not be visible immediately after creation. Retry until the snapshot is + // consistent. + readSchemaCallCount++; + for (int tries = 0; tries < 30; tries++) { + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() + .anyMatch(t -> t.getColumns().isEmpty()); + boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; + boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); + if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; } public void addFunctionAndType(String functionName, Character functionType) { From c7e8b04ebd169bc914e590b4407417138159799a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 21 Apr 2026 09:15:55 +0800 Subject: [PATCH 057/132] Refactor: extract common INSERT generation logic into AbstractInsertGenerator Move shared boilerplate (INSERT INTO, column list, ExpectedErrors field) from 12 concrete InsertGenerator subclasses into AbstractInsertGenerator, reducing ~79 lines of duplicated code. Co-Authored-By: Claude Opus 4.6 --- .../gen/ClickHouseInsertGenerator.java | 10 +--------- .../common/gen/AbstractInsertGenerator.java | 20 ++++++++++++++++++- .../databend/gen/DatabendInsertGenerator.java | 12 +---------- .../gen/DataFusionInsertGenerator.java | 15 +------------- .../doris/gen/DorisInsertGenerator.java | 11 +--------- .../duckdb/gen/DuckDBInsertGenerator.java | 11 +--------- src/sqlancer/h2/H2InsertGenerator.java | 6 +----- .../hive/gen/HiveInsertGenerator.java | 2 -- .../hsqldb/gen/HSQLDBInsertGenerator.java | 12 +---------- .../presto/gen/PrestoInsertGenerator.java | 11 +--------- .../questdb/gen/QuestDBInsertGenerator.java | 13 +----------- .../spark/gen/SparkInsertGenerator.java | 2 -- .../ycql/gen/YCQLInsertGenerator.java | 12 +---------- 13 files changed, 29 insertions(+), 108 deletions(-) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 9286185a7..593dec7c6 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -11,13 +11,11 @@ import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; public class ClickHouseInsertGenerator extends AbstractInsertGenerator { private final ClickHouseGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final ClickHouseExpressionGenerator gen; public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { @@ -42,13 +40,7 @@ private SQLQueryAdapter get() { columns = table.getRandomNonEmptyColumnSubset().stream().filter(c -> !c.isAlias() && !c.isMaterialized()) .collect(Collectors.toList()); } - sb.append("INSERT INTO "); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/common/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java index 1a0b2a997..7426656b8 100644 --- a/src/sqlancer/common/gen/AbstractInsertGenerator.java +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -1,12 +1,30 @@ package sqlancer.common.gen; import java.util.List; +import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; -public abstract class AbstractInsertGenerator { +public abstract class AbstractInsertGenerator> { protected StringBuilder sb = new StringBuilder(); + protected ExpectedErrors errors = new ExpectedErrors(); + + protected void appendColumnList(List columns) { + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + } + + protected void buildInsertInto(String tableName, List columns) { + sb.append("INSERT INTO "); + sb.append(tableName); + appendColumnList(columns); + sb.append(" VALUES "); + insertColumns(columns); + } protected void insertColumns(List columns) { for (int nrRows = 0; nrRows < Randomly.smallNumber() + 1; nrRows++) { diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 340e494ef..6a3a4c51f 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -1,12 +1,9 @@ package sqlancer.databend.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -16,7 +13,6 @@ public class DatabendInsertGenerator extends AbstractInsertGenerator { private final DatabendGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DatabendInsertGenerator(DatabendGlobalState globalState) { this.globalState = globalState; @@ -27,15 +23,9 @@ public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); DatabendErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java index 1ee00dd50..43a340731 100644 --- a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.datafusion.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; @@ -15,7 +13,6 @@ public class DataFusionInsertGenerator extends AbstractInsertGenerator { private final DataFusionGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DataFusionInsertGenerator(DataFusionGlobalState globalState) { this.globalState = globalState; @@ -26,21 +23,11 @@ public static SQLQueryAdapter getQuery(DataFusionGlobalState globalState, DataFu } private SQLQueryAdapter generate(DataFusionTable targetTable) { - // `sb` is a global `StringBuilder` for current insert query - sb.append("INSERT INTO "); - if (targetTable.getColumns().isEmpty()) { throw new IgnoreMeException(); } List columns = targetTable.getRandomNonEmptyColumnSubset(); - - sb.append(targetTable.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); // will finally call `insertValue()` to generate random value - + buildInsertInto(targetTable.getName(), columns); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java index 50dc5cdec..9407ebf43 100644 --- a/src/sqlancer/doris/gen/DorisInsertGenerator.java +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.doris.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; @@ -16,7 +14,6 @@ public class DorisInsertGenerator extends AbstractInsertGenerator { private final DorisGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DorisInsertGenerator(DorisGlobalState globalState) { this.globalState = globalState; @@ -27,15 +24,9 @@ public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyInsertColumns(); - sb.append(table.getName()); - sb.append(" ("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); DorisErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 6793d2b51..65a9c222f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.duckdb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -16,7 +14,6 @@ public class DuckDBInsertGenerator extends AbstractInsertGenerator { private final DuckDBGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DuckDBInsertGenerator(DuckDBGlobalState globalState) { this.globalState = globalState; @@ -27,15 +24,9 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); DuckDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index c4e559b0e..be715315d 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; @@ -14,7 +13,6 @@ public class H2InsertGenerator extends AbstractInsertGenerator { private final H2GlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final H2ExpressionGenerator gen; public H2InsertGenerator(H2GlobalState globalState) { @@ -39,9 +37,7 @@ private SQLQueryAdapter generate() { H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); + appendColumnList(columns); if (mergeInto && Randomly.getBoolean()) { sb.append(" KEY("); sb.append(table.getRandomNonEmptyColumnSubset().stream().map(c -> c.getName()) diff --git a/src/sqlancer/hive/gen/HiveInsertGenerator.java b/src/sqlancer/hive/gen/HiveInsertGenerator.java index 963fafbce..cd0e11df7 100644 --- a/src/sqlancer/hive/gen/HiveInsertGenerator.java +++ b/src/sqlancer/hive/gen/HiveInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hive.HiveErrors; import sqlancer.hive.HiveGlobalState; @@ -14,7 +13,6 @@ public class HiveInsertGenerator extends AbstractInsertGenerator { private final HiveGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final HiveExpressionGenerator gen; public HiveInsertGenerator(HiveGlobalState globalState) { diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index 1cc132190..16020c508 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -1,10 +1,8 @@ package sqlancer.hsqldb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; @@ -14,7 +12,6 @@ public class HSQLDBInsertGenerator extends AbstractInsertGenerator { private final HSQLDBProvider.HSQLDBGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public HSQLDBInsertGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { this.globalState = globalState; @@ -25,16 +22,9 @@ public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalSt } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); - // HSQLDBErrors.addInsertErrors(errors); + buildInsertInto(table.getName(), columns); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index 072a22ae0..e9fa06f7b 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -1,10 +1,8 @@ package sqlancer.presto.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; @@ -26,16 +24,9 @@ public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); - ExpectedErrors errors = new ExpectedErrors(); + buildInsertInto(table.getName(), columns); PrestoErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, false, false); } diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java index 754a0d2e5..e3a4dc35d 100644 --- a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -1,12 +1,9 @@ package sqlancer.questdb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; import sqlancer.questdb.QuestDBErrors; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; @@ -17,22 +14,14 @@ public class QuestDBInsertGenerator extends AbstractInsertGenerator columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); QuestDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java index b1755a848..c1404f509 100644 --- a/src/sqlancer/spark/gen/SparkInsertGenerator.java +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.spark.SparkErrors; import sqlancer.spark.SparkGlobalState; @@ -14,7 +13,6 @@ public class SparkInsertGenerator extends AbstractInsertGenerator { private final SparkGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final SparkExpressionGenerator gen; public SparkInsertGenerator(SparkGlobalState globalState) { diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java index a1159d310..d53c70d29 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -1,12 +1,9 @@ package sqlancer.yugabyte.ycql.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ycql.YCQLErrors; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; @@ -16,7 +13,6 @@ public class YCQLInsertGenerator extends AbstractInsertGenerator { private final YCQLGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public YCQLInsertGenerator(YCQLGlobalState globalState) { this.globalState = globalState; @@ -27,15 +23,9 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getColumns(); - sb.append(globalState.getDatabaseName()).append(".").append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(globalState.getDatabaseName() + "." + table.getName(), columns); errors.add("Invalid Arguments"); errors.add("Null Argument for Primary Key"); From 38d14b423a31a46adb113c938fbce777854ac968 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 21 Apr 2026 10:52:45 +0800 Subject: [PATCH 058/132] Unify INSERT/UPDATE/DELETE generators under AbstractGenerator hierarchy Introduce AbstractDeleteGenerator extending AbstractGenerator, and make AbstractInsertGenerator and AbstractUpdateGenerator also extend it, eliminating duplicate sb/errors field declarations across all generator classes. Convert all 15 DELETE generators (CockroachDB, Databend, Doris, DuckDB, H2, MariaDB, Materialize, MySQL, OceanBase, Postgres, Presto, SQLite3, TiDB, YCQL, YSQL) from static-utility classes to the instance-based buildStatement() pattern. Add canonicalizeString field to AbstractGenerator (defaulting to true) so generators like Presto/Hive/Spark can opt out of semicolon canonicalization. Co-Authored-By: Claude Sonnet 4.6 --- .../gen/ClickHouseInsertGenerator.java | 9 +++--- .../gen/CockroachDBDeleteGenerator.java | 17 ++++++---- .../gen/CockroachDBIndexGenerator.java | 2 +- .../gen/CockroachDBTableGenerator.java | 2 +- .../gen/CockroachDBUpdateGenerator.java | 6 ++-- .../common/gen/AbstractDeleteGenerator.java | 5 +++ .../common/gen/AbstractGenerator.java | 5 +-- .../common/gen/AbstractInsertGenerator.java | 6 +--- .../common/gen/AbstractUpdateGenerator.java | 6 +--- .../databend/gen/DatabendDeleteGenerator.java | 18 +++++++---- .../databend/gen/DatabendInsertGenerator.java | 6 ++-- .../gen/DataFusionInsertGenerator.java | 10 +++--- .../doris/gen/DorisDeleteGenerator.java | 18 +++++++---- .../doris/gen/DorisInsertGenerator.java | 8 ++--- .../doris/gen/DorisUpdateGenerator.java | 6 ++-- .../duckdb/gen/DuckDBDeleteGenerator.java | 18 +++++++---- .../duckdb/gen/DuckDBInsertGenerator.java | 6 ++-- .../duckdb/gen/DuckDBUpdateGenerator.java | 6 ++-- src/sqlancer/h2/H2DeleteGenerator.java | 18 +++++++---- src/sqlancer/h2/H2InsertGenerator.java | 6 ++-- src/sqlancer/h2/H2UpdateGenerator.java | 6 ++-- .../hive/gen/HiveInsertGenerator.java | 7 ++-- .../hsqldb/gen/HSQLDBInsertGenerator.java | 6 ++-- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 6 ++-- .../mariadb/gen/MariaDBDeleteGenerator.java | 25 +++++++++------ .../gen/MaterializeDeleteGenerator.java | 18 +++++++---- .../gen/MaterializeUpdateGenerator.java | 8 ++--- .../mysql/gen/MySQLDeleteGenerator.java | 12 +++---- .../mysql/gen/MySQLUpdateGenerator.java | 10 +++--- .../gen/OceanBaseDeleteGenerator.java | 12 +++---- .../gen/OceanBaseUpdateGenerator.java | 7 ++-- .../postgres/gen/PostgresDeleteGenerator.java | 18 +++++++---- .../postgres/gen/PostgresUpdateGenerator.java | 8 ++--- .../presto/gen/PrestoDeleteGenerator.java | 19 +++++++---- .../presto/gen/PrestoInsertGenerator.java | 7 ++-- .../presto/gen/PrestoUpdateGenerator.java | 7 ++-- .../questdb/gen/QuestDBInsertGenerator.java | 12 +++---- .../spark/gen/SparkInsertGenerator.java | 7 ++-- .../gen/dml/SQLite3DeleteGenerator.java | 32 ++++++++++++------- .../gen/dml/SQLite3UpdateGenerator.java | 15 +++++---- .../tidb/gen/TiDBDeleteGenerator.java | 23 +++++++------ .../tidb/gen/TiDBUpdateGenerator.java | 10 +++--- .../ycql/gen/YCQLDeleteGenerator.java | 19 +++++++---- .../ycql/gen/YCQLInsertGenerator.java | 6 ++-- .../ycql/gen/YCQLUpdateGenerator.java | 6 ++-- .../ysql/gen/YSQLDeleteGenerator.java | 18 +++++++---- .../ysql/gen/YSQLUpdateGenerator.java | 8 ++--- 47 files changed, 291 insertions(+), 219 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractDeleteGenerator.java diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 593dec7c6..3951f4b55 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -1,6 +1,5 @@ package sqlancer.clickhouse.gen; -import java.sql.SQLException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -29,11 +28,12 @@ public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { ClickHouseErrors.addExpectedExpressionErrors(errors); } - public static SQLQueryAdapter getQuery(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseInsertGenerator(globalState).get(); + public static SQLQueryAdapter getQuery(ClickHouseGlobalState globalState) { + return new ClickHouseInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter get() { + @Override + public void buildStatement() { ClickHouseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = Collections.emptyList(); while (columns.isEmpty()) { @@ -41,7 +41,6 @@ private SQLQueryAdapter get() { .collect(Collectors.toList()); } buildInsertInto(table.getName(), columns); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index 23fbae389..f0048d278 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -6,17 +6,23 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; -public final class CockroachDBDeleteGenerator { +public final class CockroachDBDeleteGenerator extends AbstractDeleteGenerator { - private CockroachDBDeleteGenerator() { + private final CockroachDBGlobalState globalState; + + private CockroachDBDeleteGenerator(CockroachDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + return new CockroachDBDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append("DELETE FROM "); sb.append(table.getName()); @@ -30,7 +36,6 @@ public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { } errors.add("foreign key violation"); CockroachDBErrors.addTransactionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 4db3588ac..da6d3c609 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -21,7 +21,7 @@ public static SQLQueryAdapter create(CockroachDBGlobalState s) { if (s.getSchema().getIndexCount() >= s.getDbmsSpecificOptions().maxNumIndexes) { throw new IgnoreMeException(); } - return new CockroachDBIndexGenerator(s).getQuery(); + return new CockroachDBIndexGenerator(s).getStatement(); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index c8bfa3c6a..8678aff4d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -32,7 +32,7 @@ public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { throw new IgnoreMeException(); } - return new CockroachDBTableGenerator(globalState).getQuery(); + return new CockroachDBTableGenerator(globalState).getStatement(); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index 8dcd605d7..b367f5c59 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -22,10 +22,11 @@ private CockroachDBUpdateGenerator(CockroachDBGlobalState globalState) { } public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { - return new CockroachDBUpdateGenerator(globalState).generate(); + return new CockroachDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); @@ -51,7 +52,6 @@ private SQLQueryAdapter generate() { errors.add("cannot write directly to computed column"); CockroachDBErrors.addExpressionErrors(errors); CockroachDBErrors.addTransactionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java new file mode 100644 index 000000000..47df974e6 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -0,0 +1,5 @@ +package sqlancer.common.gen; + +public abstract class AbstractDeleteGenerator extends AbstractGenerator { + +} diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java index c478610c4..5d13fc746 100644 --- a/src/sqlancer/common/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -8,10 +8,11 @@ public abstract class AbstractGenerator { protected final ExpectedErrors errors = new ExpectedErrors(); protected final StringBuilder sb = new StringBuilder(); protected boolean canAffectSchema; + protected boolean canonicalizeString = true; - public SQLQueryAdapter getQuery() { + public SQLQueryAdapter getStatement() { buildStatement(); - return new SQLQueryAdapter(sb.toString(), errors, canAffectSchema); + return new SQLQueryAdapter(sb.toString(), errors, canAffectSchema, canonicalizeString); } public abstract void buildStatement(); diff --git a/src/sqlancer/common/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java index 7426656b8..1a1b36b67 100644 --- a/src/sqlancer/common/gen/AbstractInsertGenerator.java +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -4,13 +4,9 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; -public abstract class AbstractInsertGenerator> { - - protected StringBuilder sb = new StringBuilder(); - protected ExpectedErrors errors = new ExpectedErrors(); +public abstract class AbstractInsertGenerator> extends AbstractGenerator { protected void appendColumnList(List columns) { sb.append("("); diff --git a/src/sqlancer/common/gen/AbstractUpdateGenerator.java b/src/sqlancer/common/gen/AbstractUpdateGenerator.java index f130c15a5..52b716bff 100644 --- a/src/sqlancer/common/gen/AbstractUpdateGenerator.java +++ b/src/sqlancer/common/gen/AbstractUpdateGenerator.java @@ -2,13 +2,9 @@ import java.util.List; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; -public abstract class AbstractUpdateGenerator> { - - protected final ExpectedErrors errors = new ExpectedErrors(); - protected StringBuilder sb = new StringBuilder(); +public abstract class AbstractUpdateGenerator> extends AbstractGenerator { protected void updateColumns(List columns) { for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index 256030409..5adabf175 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -1,21 +1,28 @@ package sqlancer.databend.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; -public final class DatabendDeleteGenerator { +public final class DatabendDeleteGenerator extends AbstractDeleteGenerator { - private DatabendDeleteGenerator() { + private final DatabendGlobalState globalState; + + private DatabendDeleteGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(DatabendGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new DatabendDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); @@ -23,7 +30,6 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN))); DatabendErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 6a3a4c51f..13ee80e95 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -19,15 +19,15 @@ public DatabendInsertGenerator(DatabendGlobalState globalState) { } public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - return new DatabendInsertGenerator(globalState).generate(); + return new DatabendInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); DatabendErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java index 43a340731..36a178791 100644 --- a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java @@ -13,22 +13,24 @@ public class DataFusionInsertGenerator extends AbstractInsertGenerator { private final DataFusionGlobalState globalState; + private final DataFusionTable targetTable; - public DataFusionInsertGenerator(DataFusionGlobalState globalState) { + public DataFusionInsertGenerator(DataFusionGlobalState globalState, DataFusionTable targetTable) { this.globalState = globalState; + this.targetTable = targetTable; } public static SQLQueryAdapter getQuery(DataFusionGlobalState globalState, DataFusionTable targetTable) { - return new DataFusionInsertGenerator(globalState).generate(targetTable); + return new DataFusionInsertGenerator(globalState, targetTable).getStatement(); } - private SQLQueryAdapter generate(DataFusionTable targetTable) { + @Override + public void buildStatement() { if (targetTable.getColumns().isEmpty()) { throw new IgnoreMeException(); } List columns = targetTable.getRandomNonEmptyColumnSubset(); buildInsertInto(targetTable.getName(), columns); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/doris/gen/DorisDeleteGenerator.java b/src/sqlancer/doris/gen/DorisDeleteGenerator.java index 27f369aec..66deaea8f 100644 --- a/src/sqlancer/doris/gen/DorisDeleteGenerator.java +++ b/src/sqlancer/doris/gen/DorisDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.doris.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; @@ -9,14 +9,21 @@ import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.visitor.DorisToStringVisitor; -public final class DorisDeleteGenerator { +public final class DorisDeleteGenerator extends AbstractDeleteGenerator { - private DorisDeleteGenerator() { + private final DorisGlobalState globalState; + + private DorisDeleteGenerator(DorisGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(DorisGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new DorisDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -25,7 +32,6 @@ public static SQLQueryAdapter generate(DorisGlobalState globalState) { .setColumns(table.getColumns()).generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java index 9407ebf43..e05bc85fd 100644 --- a/src/sqlancer/doris/gen/DorisInsertGenerator.java +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -20,15 +20,15 @@ public DorisInsertGenerator(DorisGlobalState globalState) { } public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { - return new DorisInsertGenerator(globalState).generate(); + return new DorisInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyInsertColumns(); + List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); DorisErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java index 906173921..e0db2b7b4 100644 --- a/src/sqlancer/doris/gen/DorisUpdateGenerator.java +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -23,10 +23,11 @@ private DorisUpdateGenerator(DorisGlobalState globalState) { } public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { - return new DorisUpdateGenerator(globalState).generate(); + return new DorisUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new DorisNewExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -37,7 +38,6 @@ private SQLQueryAdapter generate() { sb.append(" WHERE "); sb.append(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index 5fadce30a..42695a9f6 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -1,21 +1,28 @@ package sqlancer.duckdb.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBToStringVisitor; -public final class DuckDBDeleteGenerator { +public final class DuckDBDeleteGenerator extends AbstractDeleteGenerator { - private DuckDBDeleteGenerator() { + private final DuckDBGlobalState globalState; + + private DuckDBDeleteGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new DuckDBDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -24,7 +31,6 @@ public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } DuckDBErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 65a9c222f..e8b122a51 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -20,15 +20,15 @@ public DuckDBInsertGenerator(DuckDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - return new DuckDBInsertGenerator(globalState).generate(); + return new DuckDBInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); buildInsertInto(table.getName(), columns); DuckDBErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index b4ffd0140..8e2ddd047 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -22,10 +22,11 @@ private DuckDBUpdateGenerator(DuckDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - return new DuckDBUpdateGenerator(globalState).generate(); + return new DuckDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -34,7 +35,6 @@ private SQLQueryAdapter generate() { sb.append(" SET "); updateColumns(columns); DuckDBErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java index 58afdcb48..dd0fdff34 100644 --- a/src/sqlancer/h2/H2DeleteGenerator.java +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -1,19 +1,26 @@ package sqlancer.h2; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; -public final class H2DeleteGenerator { +public final class H2DeleteGenerator extends AbstractDeleteGenerator { - private H2DeleteGenerator() { + private final H2GlobalState globalState; + + private H2DeleteGenerator(H2GlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new H2DeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -27,7 +34,6 @@ public static SQLQueryAdapter getQuery(H2GlobalState globalState) { } H2Errors.addExpressionErrors(errors); H2Errors.addDeleteErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index be715315d..3bd7552d8 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -21,10 +21,11 @@ public H2InsertGenerator(H2GlobalState globalState) { } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - return new H2InsertGenerator(globalState).generate(); + return new H2InsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { boolean mergeInto = false; // Randomly.getBooleanWithRatherLowProbability(); if (mergeInto) { sb.append("MERGE INTO "); @@ -48,7 +49,6 @@ private SQLQueryAdapter generate() { insertColumns(columns); H2Errors.addInsertErrors(errors); H2Errors.addExpressionErrors(errors); // generated columns - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index 158621409..fe63c7a4e 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -19,10 +19,11 @@ private H2UpdateGenerator(H2GlobalState globalState) { } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - return new H2UpdateGenerator(globalState).generate(); + return new H2UpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new H2ExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -37,7 +38,6 @@ private SQLQueryAdapter generate() { sb.append(H2ToStringVisitor.asString(gen.generateExpression())); } H2Errors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/hive/gen/HiveInsertGenerator.java b/src/sqlancer/hive/gen/HiveInsertGenerator.java index cd0e11df7..8c23e4456 100644 --- a/src/sqlancer/hive/gen/HiveInsertGenerator.java +++ b/src/sqlancer/hive/gen/HiveInsertGenerator.java @@ -18,10 +18,11 @@ public class HiveInsertGenerator extends AbstractInsertGenerator { public HiveInsertGenerator(HiveGlobalState globalState) { this.globalState = globalState; this.gen = new HiveExpressionGenerator(globalState); + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(HiveGlobalState globalState) { - return new HiveInsertGenerator(globalState).generate(); + return new HiveInsertGenerator(globalState).getStatement(); } @Override @@ -29,7 +30,8 @@ protected void insertValue(HiveColumn column) { sb.append(HiveToStringVisitor.asString(gen.generateConstant())); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { // Inserting values into tables from SQL. sb.append("INSERT INTO "); HiveTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); @@ -45,6 +47,5 @@ private SQLQueryAdapter generate() { insertColumns(columns); HiveErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index 16020c508..00d99be38 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -18,14 +18,14 @@ public HSQLDBInsertGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { - return new HSQLDBInsertGenerator(globalState).generate(); + return new HSQLDBInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index e639e21b3..54380214f 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -24,10 +24,11 @@ private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { - return new HSQLDBUpdateGenerator(globalState).generate(); + return new HSQLDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -42,7 +43,6 @@ private SQLQueryAdapter generate() { errors.add("data type of expression is not boolean"); HSQLDBErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index 6d85eb891..2992f569d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -3,7 +3,7 @@ import java.util.Collections; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTables; import sqlancer.mariadb.MariaDBSchema; @@ -11,12 +11,22 @@ import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.ast.MariaDBVisitor; -public final class MariaDBDeleteGenerator { +public final class MariaDBDeleteGenerator extends AbstractDeleteGenerator { - private MariaDBDeleteGenerator() { + private final MariaDBSchema schema; + private final Randomly r; + + private MariaDBDeleteGenerator(MariaDBSchema schema, Randomly r) { + this.schema = schema; + this.r = r; } public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { + return new MariaDBDeleteGenerator(schema, r).getStatement(); + } + + @Override + public void buildStatement() { MariaDBTable table = schema.getRandomTable(); MariaDBExpressionGenerator expressionGenerator = new MariaDBExpressionGenerator(r); @@ -25,15 +35,13 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { Collections.singletonList(table)); expressionGenerator.setTablesAndColumns(tablesAndColumns); - ExpectedErrors errors = new ExpectedErrors(); - errors.add("foreign key constraint fails"); errors.add("cannot delete or update a parent row"); errors.add("Data truncated"); errors.add("Division by 0"); errors.add("Incorrect value"); - StringBuilder sb = new StringBuilder("DELETE"); + sb.append("DELETE"); if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" LOW_PRIORITY"); @@ -81,13 +89,10 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { } } - String query = sb.toString(); - if (query.contains("RLIKE") || query.contains("REGEXP")) { + if (sb.toString().contains("RLIKE") || sb.toString().contains("REGEXP")) { errors.add("Regex error"); errors.add("quantifier does not follow a repeatable item"); errors.add("Got error"); } - - return new SQLQueryAdapter(query, errors); } } diff --git a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java index ded3e53f4..2aceddcc9 100644 --- a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java @@ -1,25 +1,32 @@ package sqlancer.materialize.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.materialize.MaterializeGlobalState; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; import sqlancer.materialize.MaterializeSchema.MaterializeTable; import sqlancer.materialize.MaterializeVisitor; -public final class MaterializeDeleteGenerator { +public final class MaterializeDeleteGenerator extends AbstractDeleteGenerator { - private MaterializeDeleteGenerator() { + private final MaterializeGlobalState globalState; + + private MaterializeDeleteGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + return new MaterializeDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { MaterializeTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append("DELETE FROM"); sb.append(" "); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -32,7 +39,6 @@ public static SQLQueryAdapter create(MaterializeGlobalState globalState) { errors.add("does not support casting"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java index 1b7e69208..7d338c027 100644 --- a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java @@ -20,6 +20,7 @@ public final class MaterializeUpdateGenerator extends AbstractUpdateGenerator t.isInsertable()); List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); @@ -55,8 +57,6 @@ private SQLQueryAdapter generate() { randomTable.getColumns(), MaterializeDataType.BOOLEAN); sb.append(MaterializeVisitor.asString(where)); } - - return new SQLQueryAdapter(sb.toString(), errors, true); } @Override diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index f12c23a72..048ef335f 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -3,16 +3,15 @@ import java.util.Arrays; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; -public class MySQLDeleteGenerator { +public class MySQLDeleteGenerator extends AbstractDeleteGenerator { - private final StringBuilder sb = new StringBuilder(); private final MySQLGlobalState globalState; public MySQLDeleteGenerator(MySQLGlobalState globalState) { @@ -20,13 +19,13 @@ public MySQLDeleteGenerator(MySQLGlobalState globalState) { } public static SQLQueryAdapter delete(MySQLGlobalState globalState) { - return new MySQLDeleteGenerator(globalState).generate(); + return new MySQLDeleteGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { MySQLTable randomTable = globalState.getSchema().getRandomTable(); MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()); - ExpectedErrors errors = new ExpectedErrors(); sb.append("DELETE"); if (Randomly.getBoolean()) { sb.append(" LOW_PRIORITY"); @@ -51,7 +50,6 @@ private SQLQueryAdapter generate() { */, "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index")); // TODO: support ORDER BY - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java index 55ba3dd45..1aca84580 100644 --- a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java @@ -1,6 +1,5 @@ package sqlancer.mysql.gen; -import java.sql.SQLException; import java.util.List; import sqlancer.Randomly; @@ -21,11 +20,12 @@ public MySQLUpdateGenerator(MySQLGlobalState globalState) { this.globalState = globalState; } - public static SQLQueryAdapter create(MySQLGlobalState globalState) throws SQLException { - return new MySQLUpdateGenerator(globalState).generate(); + public static SQLQueryAdapter create(MySQLGlobalState globalState) { + return new MySQLUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() throws SQLException { + @Override + public void buildStatement() { MySQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new MySQLExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -40,8 +40,6 @@ private SQLQueryAdapter generate() throws SQLException { } MySQLErrors.addInsertUpdateErrors(errors); errors.add("doesn't have this option"); - - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index ea1cb36e8..c46ace304 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -3,16 +3,15 @@ import java.util.Arrays; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; import sqlancer.oceanbase.OceanBaseVisitor; -public class OceanBaseDeleteGenerator { +public class OceanBaseDeleteGenerator extends AbstractDeleteGenerator { - private final StringBuilder sb = new StringBuilder(); private final OceanBaseGlobalState globalState; private final Randomly r; @@ -22,14 +21,14 @@ public OceanBaseDeleteGenerator(OceanBaseGlobalState globalState) { } public static SQLQueryAdapter delete(OceanBaseGlobalState globalState) { - return new OceanBaseDeleteGenerator(globalState).generate(); + return new OceanBaseDeleteGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { OceanBaseTable randomTable = globalState.getSchema().getRandomTable(); OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState) .setColumns(randomTable.getColumns()); - ExpectedErrors errors = new ExpectedErrors(); sb.append("DELETE"); if (Randomly.getBoolean()) { sb.append(" /*+parallel(" + r.getLong(0, 10) + ") enable_parallel_dml*/ "); @@ -45,7 +44,6 @@ private SQLQueryAdapter generate() { "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index", "Incorrect value", "Out of range value for column", "Data truncation:")); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index 950317bd2..44fdf39fe 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -23,10 +23,11 @@ public OceanBaseUpdateGenerator(OceanBaseGlobalState globalState) { } public static SQLQueryAdapter update(OceanBaseGlobalState globalState) { - return new OceanBaseUpdateGenerator(globalState).generate(); + return new OceanBaseUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -45,8 +46,6 @@ private SQLQueryAdapter generate() { } errors.add("Duplicated primary key"); OceanBaseErrors.addInsertErrors(errors); - - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index f827331c0..452f5cdf4 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -1,25 +1,32 @@ package sqlancer.postgres.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; -public final class PostgresDeleteGenerator { +public final class PostgresDeleteGenerator extends AbstractDeleteGenerator { - private PostgresDeleteGenerator() { + private final PostgresGlobalState globalState; + + private PostgresDeleteGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter create(PostgresGlobalState globalState) { + return new PostgresDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append("DELETE FROM"); if (Randomly.getBoolean()) { sb.append(" ONLY"); } @@ -40,7 +47,6 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { errors.add("cannot cast"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 7ce7fe882..1effad582 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -20,6 +20,7 @@ public final class PostgresUpdateGenerator extends AbstractUpdateGenerator t.isInsertable()); List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); @@ -55,8 +57,6 @@ private SQLQueryAdapter generate() { randomTable.getColumns(), PostgresDataType.BOOLEAN); sb.append(PostgresVisitor.asString(where)); } - - return new SQLQueryAdapter(sb.toString(), errors, true); } @Override diff --git a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java index 9f869c241..59b7b4174 100644 --- a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java +++ b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.presto.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; @@ -9,14 +9,22 @@ import sqlancer.presto.PrestoSchema.PrestoTable; import sqlancer.presto.PrestoToStringVisitor; -public final class PrestoDeleteGenerator { +public final class PrestoDeleteGenerator extends AbstractDeleteGenerator { - private PrestoDeleteGenerator() { + private final PrestoGlobalState globalState; + + private PrestoDeleteGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canonicalizeString = false; } public static SQLQueryAdapter generate(PrestoGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new PrestoDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -26,7 +34,6 @@ public static SQLQueryAdapter generate(PrestoGlobalState globalState) { .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()))); } PrestoErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } } diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index e9fa06f7b..15d5f3543 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -17,18 +17,19 @@ public class PrestoInsertGenerator extends AbstractInsertGenerator public PrestoInsertGenerator(PrestoGlobalState globalState) { this.globalState = globalState; + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { - return new PrestoInsertGenerator(globalState).generate(); + return new PrestoInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); PrestoErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } @Override diff --git a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java index a8afcc578..3c197f0b9 100644 --- a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java +++ b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java @@ -19,13 +19,15 @@ public final class PrestoUpdateGenerator extends AbstractUpdateGenerator !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -34,7 +36,6 @@ private SQLQueryAdapter generate() { sb.append(" SET "); updateColumns(columns); PrestoErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } @Override diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java index e3a4dc35d..41ba72406 100644 --- a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -18,16 +18,16 @@ public QuestDBInsertGenerator(QuestDBGlobalState globalState) { this.globalState = globalState; } - private SQLQueryAdapter generate() { + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + return new QuestDBInsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { QuestDBTable table = globalState.getSchema().getRandomTable(); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); QuestDBErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); - } - - public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { - return new QuestDBInsertGenerator(globalState).generate(); } @Override diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java index c1404f509..c43584315 100644 --- a/src/sqlancer/spark/gen/SparkInsertGenerator.java +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -18,10 +18,11 @@ public class SparkInsertGenerator extends AbstractInsertGenerator { public SparkInsertGenerator(SparkGlobalState globalState) { this.globalState = globalState; this.gen = new SparkExpressionGenerator(globalState); + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(SparkGlobalState globalState) { - return new SparkInsertGenerator(globalState).generate(); + return new SparkInsertGenerator(globalState).getStatement(); } @Override @@ -29,7 +30,8 @@ protected void insertValue(SparkColumn column) { sb.append(SparkToStringVisitor.asString(gen.generateConstant())); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { sb.append("INSERT INTO "); SparkTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); @@ -40,6 +42,5 @@ private SQLQueryAdapter generate() { insertColumns(columns); SparkErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 4c86417d0..51b23eaa4 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -3,7 +3,7 @@ import java.util.Arrays; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; @@ -11,26 +11,35 @@ import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public final class SQLite3DeleteGenerator { +public final class SQLite3DeleteGenerator extends AbstractDeleteGenerator { - private SQLite3DeleteGenerator() { + private final SQLite3GlobalState globalState; + private final SQLite3Table table; + + private SQLite3DeleteGenerator(SQLite3GlobalState globalState, SQLite3Table table) { + this.globalState = globalState; + this.table = table; + this.canAffectSchema = true; } public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState) { - SQLite3Table tableName = globalState.getSchema().getRandomTable(t -> !t.isView() && !t.isReadOnly()); - return deleteContent(globalState, tableName); + SQLite3Table table = globalState.getSchema().getRandomTable(t -> !t.isView() && !t.isReadOnly()); + return deleteContent(globalState, table); + } + + public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLite3Table table) { + return new SQLite3DeleteGenerator(globalState, table).getStatement(); } - public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLite3Table tableName) { - StringBuilder sb = new StringBuilder(); + @Override + public void buildStatement() { sb.append("DELETE FROM "); - sb.append(tableName.getName()); + sb.append(table.getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); - sb.append(SQLite3Visitor.asString(new SQLite3ExpressionGenerator(globalState) - .setColumns(tableName.getColumns()).generateExpression())); + sb.append(SQLite3Visitor.asString( + new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } - ExpectedErrors errors = new ExpectedErrors(); SQLite3Errors.addExpectedExpressionErrors(errors); errors.addAll(Arrays.asList("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", "[SQLITE_CONSTRAINT] Abort due to constraint violation ", @@ -40,7 +49,6 @@ public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLi "cannot INSERT into generated column", "A table in the database is locked", "load_extension() prohibited in triggers and views", "The database file is locked")); SQLite3Errors.addDeleteErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 5a17ad339..9a2be3150 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -18,10 +18,13 @@ public class SQLite3UpdateGenerator extends AbstractUpdateGenerator columnsToUpdate = Randomly.nonEmptySubsetPotentialDuplicates(table.getColumns()); sb.append("UPDATE "); if (Randomly.getBoolean()) { @@ -98,8 +101,6 @@ private SQLQueryAdapter generate(SQLite3Table table) { SQLite3Errors.addInsertNowErrors(errors); SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addDeleteErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true /* column could have an ON UPDATE clause */); - } @Override diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index c3986f8d9..cbe5fc721 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -1,10 +1,9 @@ package sqlancer.tidb.gen; -import java.sql.SQLException; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; @@ -12,16 +11,24 @@ import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.visitor.TiDBVisitor; -public final class TiDBDeleteGenerator { +public final class TiDBDeleteGenerator extends AbstractDeleteGenerator { - private TiDBDeleteGenerator() { + private final TiDBGlobalState globalState; + + private TiDBDeleteGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + return new TiDBDeleteGenerator(globalState).getStatement(); } - public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); + @Override + public void buildStatement() { + errors.addAll(TiDBErrors.getExpressionErrors()); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); - StringBuilder sb = new StringBuilder("DELETE "); + sb.append("DELETE "); if (Randomly.getBooleanWithSmallProbability()) { sb.append("LOW_PRIORITY "); } @@ -55,8 +62,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx errors.add("is not valid for CHARACTER SET"); errors.add("Division by 0"); errors.add("error parsing regexp"); - return new SQLQueryAdapter(sb.toString(), errors); - } } diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 241ee3321..af6430b48 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -1,6 +1,5 @@ package sqlancer.tidb.gen; -import java.sql.SQLException; import java.util.List; import sqlancer.Randomly; @@ -22,11 +21,12 @@ private TiDBUpdateGenerator(TiDBGlobalState globalState) { this.globalState = globalState; } - public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - return new TiDBUpdateGenerator(globalState).generate(); + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + return new TiDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() throws SQLException { + @Override + public void buildStatement() { TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -40,8 +40,6 @@ private SQLQueryAdapter generate() throws SQLException { sb.append(TiDBVisitor.asString(gen.generateExpression())); } TiDBErrors.addInsertErrors(errors); - - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java index 108cd1be9..6ebe0db5f 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java @@ -1,31 +1,36 @@ package sqlancer.yugabyte.ycql.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLErrors; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; import sqlancer.yugabyte.ycql.YCQLToStringVisitor; -public final class YCQLDeleteGenerator { +public final class YCQLDeleteGenerator extends AbstractDeleteGenerator { - private YCQLDeleteGenerator() { + private final YCQLGlobalState globalState; + + private YCQLDeleteGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(YCQLGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new YCQLDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append("DELETE FROM "); sb.append(table.getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append(YCQLToStringVisitor.asString( new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } - YCQLErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java index d53c70d29..167f5d237 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -19,10 +19,11 @@ public YCQLInsertGenerator(YCQLGlobalState globalState) { } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - return new YCQLInsertGenerator(globalState).generate(); + return new YCQLInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getColumns(); buildInsertInto(globalState.getDatabaseName() + "." + table.getName(), columns); @@ -31,7 +32,6 @@ private SQLQueryAdapter generate() { errors.add("Null Argument for Primary Key"); YCQLErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java index a6c855cf3..eee49bee4 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -22,10 +22,11 @@ private YCQLUpdateGenerator(YCQLGlobalState globalState) { } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - return new YCQLUpdateGenerator(globalState).generate(); + return new YCQLUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -41,7 +42,6 @@ private SQLQueryAdapter generate() { errors.add("Missing Argument for Primary Key"); YCQLErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java index e0128707b..6e35c6862 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.yugabyte.ysql.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; @@ -9,18 +9,25 @@ import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; import sqlancer.yugabyte.ysql.YSQLVisitor; -public final class YSQLDeleteGenerator { +public final class YSQLDeleteGenerator extends AbstractDeleteGenerator { - private YSQLDeleteGenerator() { + private final YSQLGlobalState globalState; + + private YSQLDeleteGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter create(YSQLGlobalState globalState) { + return new YSQLDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { YSQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append("DELETE FROM"); if (Randomly.getBoolean()) { sb.append(" ONLY"); } @@ -41,7 +48,6 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { errors.add("cannot cast"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index bc7b00d79..6c5fd4144 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -21,6 +21,7 @@ public final class YSQLUpdateGenerator extends AbstractUpdateGenerator columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); @@ -57,8 +59,6 @@ private SQLQueryAdapter generate() { YSQLDataType.BOOLEAN); sb.append(YSQLVisitor.asString(where)); } - - return new SQLQueryAdapter(sb.toString(), errors, true); } @Override From 5d2e48a932d0fa17f0fc32ab4bd088635ca426b7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Apr 2026 00:35:56 +0800 Subject: [PATCH 059/132] Address review feedback: fix comment and add MaterializeBugs Remove incorrect claim about eventual consistency in readSchema comment. Add MaterializeBugs class with bugSchemaReadIncomplete flag to guard the retry logic, following the MySQLBugs pattern. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/materialize/MaterializeBugs.java | 12 ++++++++++++ src/sqlancer/materialize/MaterializeGlobalState.java | 8 +++++--- 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 src/sqlancer/materialize/MaterializeBugs.java diff --git a/src/sqlancer/materialize/MaterializeBugs.java b/src/sqlancer/materialize/MaterializeBugs.java new file mode 100644 index 000000000..a7611a925 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeBugs.java @@ -0,0 +1,12 @@ +package sqlancer.materialize; + +// do not make the fields final to avoid warnings +public final class MaterializeBugs { + + // Tables or columns may be missing when reading information_schema shortly after creation + public static boolean bugSchemaReadIncomplete = true; + + private MaterializeBugs() { + } + +} diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java index 8a66a7819..46977529b 100644 --- a/src/sqlancer/materialize/MaterializeGlobalState.java +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -268,10 +268,12 @@ public String getRandomTableAccessMethod() { @Override public MaterializeSchema readSchema() throws SQLException { - // Materialize's information_schema is eventually consistent: tables and columns - // may not be visible immediately after creation. Retry until the snapshot is - // consistent. + // Workaround for a suspected Materialize bug where tables or columns may be + // missing when reading the schema; retry until stable. readSchemaCallCount++; + if (!MaterializeBugs.bugSchemaReadIncomplete) { + return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + } for (int tries = 0; tries < 30; tries++) { MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() From 82612b1d0f310cc131661d2b92a8d8571453a2f9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Apr 2026 22:16:13 +0800 Subject: [PATCH 060/132] Wrap readSchema workaround in bugSchemaReadIncomplete guard Co-Authored-By: Claude Opus 4.7 --- .../materialize/MaterializeGlobalState.java | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java index 46977529b..7ab963c03 100644 --- a/src/sqlancer/materialize/MaterializeGlobalState.java +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -268,32 +268,32 @@ public String getRandomTableAccessMethod() { @Override public MaterializeSchema readSchema() throws SQLException { - // Workaround for a suspected Materialize bug where tables or columns may be - // missing when reading the schema; retry until stable. - readSchemaCallCount++; - if (!MaterializeBugs.bugSchemaReadIncomplete) { - return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); - } - for (int tries = 0; tries < 30; tries++) { - MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); - boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() - .anyMatch(t -> t.getColumns().isEmpty()); - boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; - boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); - if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { - lastKnownTableCount = schema.getDatabaseTables().size(); - return schema; - } - try { - Thread.sleep(100); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; + if (MaterializeBugs.bugSchemaReadIncomplete) { + // Workaround for a suspected Materialize bug where tables or columns may be + // missing when reading the schema; retry until stable. + readSchemaCallCount++; + for (int tries = 0; tries < 30; tries++) { + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() + .anyMatch(t -> t.getColumns().isEmpty()); + boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; + boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); + if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } } + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; } - MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); - lastKnownTableCount = schema.getDatabaseTables().size(); - return schema; + return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); } public void addFunctionAndType(String functionName, Character functionType) { From 0eea0212a9a07ec7f6b3f9bf5ae1fe5a101826dd Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Apr 2026 00:49:07 +0800 Subject: [PATCH 061/132] Fix checkstyle: declare SQLite3UpdateGenerator as final Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/common/gen/AbstractDeleteGenerator.java | 3 +++ src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java index 47df974e6..8dfb0b0c6 100644 --- a/src/sqlancer/common/gen/AbstractDeleteGenerator.java +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -2,4 +2,7 @@ public abstract class AbstractDeleteGenerator extends AbstractGenerator { + protected AbstractDeleteGenerator() { + } + } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 9a2be3150..05f4b9a2e 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -14,7 +14,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3UpdateGenerator extends AbstractUpdateGenerator { +public final class SQLite3UpdateGenerator extends AbstractUpdateGenerator { private final SQLite3GlobalState globalState; private final Randomly r; From d4ed473fb7d9ba1237a168144b22b6aeb5c9cf6f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Apr 2026 10:17:37 +0800 Subject: [PATCH 062/132] Refactor: extract common CREATE INDEX logic into AbstractIndexGenerator Introduce AbstractIndexGenerator with appendCreateIndex(boolean) and appendIndexColumnList(List, boolean) helpers, and convert nine concrete index generators (DuckDB, Doris, Presto, YCQL, TiDB, MariaDB, Materialize, Postgres, YSQL) from static-utility classes to instance-based subclasses whose logic lives in buildStatement(). Public entry points (getQuery/generate) are preserved so callers in the provider classes don't need to change. --- .../common/gen/AbstractIndexGenerator.java | 33 +++++++++++++ .../doris/gen/DorisIndexGenerator.java | 31 ++++++------ .../duckdb/gen/DuckDBIndexGenerator.java | 25 ++++++---- .../mariadb/gen/MariaDBIndexGenerator.java | 48 +++++++------------ .../gen/MaterializeIndexGenerator.java | 24 ++++++---- .../postgres/gen/PostgresIndexGenerator.java | 23 ++++----- .../presto/gen/PrestoIndexGenerator.java | 26 ++++++---- src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 26 +++++----- .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 37 +++++++------- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 23 ++++----- 10 files changed, 167 insertions(+), 129 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractIndexGenerator.java diff --git a/src/sqlancer/common/gen/AbstractIndexGenerator.java b/src/sqlancer/common/gen/AbstractIndexGenerator.java new file mode 100644 index 000000000..bfa62a36c --- /dev/null +++ b/src/sqlancer/common/gen/AbstractIndexGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractIndexGenerator> extends AbstractGenerator { + + protected void appendCreateIndex(boolean unique) { + sb.append("CREATE "); + if (unique) { + sb.append("UNIQUE "); + } + sb.append("INDEX "); + } + + protected void appendIndexColumnList(List columns, boolean allowOrdering) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + if (allowOrdering && Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/doris/gen/DorisIndexGenerator.java b/src/sqlancer/doris/gen/DorisIndexGenerator.java index 5e56bb192..308c09c3f 100644 --- a/src/sqlancer/doris/gen/DorisIndexGenerator.java +++ b/src/sqlancer/doris/gen/DorisIndexGenerator.java @@ -5,42 +5,45 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisTable; -public final class DorisIndexGenerator { +public class DorisIndexGenerator extends AbstractIndexGenerator { - private DorisIndexGenerator() { + private final DorisGlobalState globalState; + + public DorisIndexGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(DorisGlobalState globalState) throws SQLException { if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { throw new IgnoreMeException(); } - ExpectedErrors errors = new ExpectedErrors(); + return new DorisIndexGenerator(globalState).getStatement(); + } + @Override + public void buildStatement() { DorisTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - String indexName = globalState.getSchema().getFreeIndexName(); - StringBuilder sb = new StringBuilder("CREATE "); - sb.append("INDEX "); + appendCreateIndex(false); if (Randomly.getBoolean()) { sb.append("IF NOT EXISTS "); } - sb.append(indexName); + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); sb.append(randomTable.getName()); - sb.append("("); - int nr = 1; // Doris Only support CREATE_INDEX on single column and index type is BITMAP; - List subset = Randomly.extractNrRandomColumns(randomTable.getColumns(), nr); - sb.append(subset.get(0).getName()); - sb.append(") "); + // Doris only supports CREATE INDEX on a single column; index type is BITMAP + List subset = Randomly.extractNrRandomColumns(randomTable.getColumns(), 1); + appendIndexColumnList(subset, false); + sb.append(" "); if (Randomly.getBoolean()) { sb.append("USING BITMAP "); } - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 6c50b204d..bd42b64bd 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -3,26 +3,32 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -public final class DuckDBIndexGenerator { +public class DuckDBIndexGenerator extends AbstractIndexGenerator { - private DuckDBIndexGenerator() { + private final DuckDBGlobalState globalState; + + public DuckDBIndexGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { + return new DuckDBIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Data contains duplicates on indexed column(s)"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); @@ -43,7 +49,6 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("cannot create an index on the rowid"); } - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 1ba3fbd4d..1fb6f10b3 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -1,31 +1,36 @@ package sqlancer.mariadb.gen; -import java.util.List; - import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -public final class MariaDBIndexGenerator { +public class MariaDBIndexGenerator extends AbstractIndexGenerator { + + private final MariaDBSchema schema; - private MariaDBIndexGenerator() { + public MariaDBIndexGenerator(MariaDBSchema schema) { + this.schema = schema; + this.canAffectSchema = true; } public static SQLQueryAdapter generate(MariaDBSchema s) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder("CREATE "); + return new MariaDBIndexGenerator(s).getStatement(); + } + + @Override + public void buildStatement() { errors.add("Key/Index cannot be defined on a virtual generated column"); errors.add("Specified key was too long"); - if (Randomly.getBoolean()) { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Duplicate entry"); errors.add("Key/Index cannot be defined on a virtual generated column"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append("i"); sb.append(DBMSCommon.createColumnName(Randomly.smallNumber())); if (Randomly.getBoolean()) { @@ -34,28 +39,9 @@ public static SQLQueryAdapter generate(MariaDBSchema s) { } sb.append(" ON "); - MariaDBTable randomTable = s.getRandomTable(); + MariaDBTable randomTable = schema.getRandomTable(); sb.append(randomTable.getName()); - sb.append("("); - List columns = Randomly.nonEmptySubset(randomTable.getColumns()); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - if (Randomly.getBoolean()) { - sb.append(" "); - sb.append(Randomly.fromOptions("ASC", "DESC")); - } - } - sb.append(")"); - // if (Randomly.getBoolean()) { - // sb.append(" ALGORITHM="); - // sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY", "NOCOPY", "INSTANT")); - // errors.add("is not supported for this operation"); - // } - - return new SQLQueryAdapter(sb.toString(), errors, true); + appendIndexColumnList(Randomly.nonEmptySubset(randomTable.getColumns()), true); } } diff --git a/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java index 9d7a91b50..a63353f1f 100644 --- a/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java @@ -1,14 +1,18 @@ package sqlancer.materialize.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeTable; -public final class MaterializeIndexGenerator { +public class MaterializeIndexGenerator extends AbstractIndexGenerator { - private MaterializeIndexGenerator() { + private final MaterializeGlobalState globalState; + + public MaterializeIndexGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; } public enum IndexType { @@ -16,17 +20,18 @@ public enum IndexType { } public static SQLQueryAdapter generate(MaterializeGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - sb.append(" INDEX "); + return new MaterializeIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(false); MaterializeTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized // views sb.append(MaterializeCommon.getFreeIndexName(globalState.getSchema())); sb.append(" ON "); sb.append(randomTable.getName()); - IndexType method; - method = IndexType.BTREE; + IndexType method = IndexType.BTREE; sb.append("("); if (method == IndexType.HASH) { @@ -75,6 +80,5 @@ public static SQLQueryAdapter generate(MaterializeGlobalState globalState) { errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); MaterializeCommon.addCommonExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 4cb2b8e3a..bf70c32d4 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -15,9 +15,12 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public final class PostgresIndexGenerator { +public class PostgresIndexGenerator extends AbstractIndexGenerator { - private PostgresIndexGenerator() { + private final PostgresGlobalState globalState; + + public PostgresIndexGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; } public enum IndexType { @@ -25,13 +28,12 @@ public enum IndexType { } public static SQLQueryAdapter generate(PostgresGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - if (Randomly.getBoolean()) { - sb.append(" UNIQUE"); - } - sb.append(" INDEX "); + return new PostgresIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(Randomly.getBoolean()); /* * Commented out as a workaround for https://www.postgresql.org/message-id/CA%2Bu7OA4XYhc- * qyCgJqwwgMGZDWAyeH821oa5oMzm_HEifZ4BeA%40mail.gmail.com @@ -136,7 +138,6 @@ public static SQLQueryAdapter generate(PostgresGlobalState globalState) { errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); PostgresCommon.addCommonExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } private static String getNewIndexName(PostgresTable randomTable) { diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java index c76283ee4..5ec23c773 100644 --- a/src/sqlancer/presto/gen/PrestoIndexGenerator.java +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -3,7 +3,7 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; @@ -12,20 +12,27 @@ import sqlancer.presto.PrestoToStringVisitor; import sqlancer.presto.ast.PrestoExpression; -public final class PrestoIndexGenerator { +public class PrestoIndexGenerator extends AbstractIndexGenerator { - private PrestoIndexGenerator() { + private final PrestoGlobalState globalState; + + public PrestoIndexGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { + return new PrestoIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information sb.append(" ON "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); @@ -50,7 +57,6 @@ public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { sb.append(PrestoToStringVisitor.asString(expr)); } errors.add("already exists!"); - return new SQLQueryAdapter(sb.toString(), errors, true, false); } } diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index 1be2753b6..64b4c808d 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -5,34 +5,39 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; import sqlancer.tidb.TiDBSchema.TiDBTable; -public final class TiDBIndexGenerator { +public class TiDBIndexGenerator extends AbstractIndexGenerator { - private TiDBIndexGenerator() { + private final TiDBGlobalState globalState; + + public TiDBIndexGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { throw new IgnoreMeException(); } - ExpectedErrors errors = new ExpectedErrors(); + return new TiDBIndexGenerator(globalState).getStatement(); + } + @Override + public void buildStatement() { TiDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - String indexName = globalState.getSchema().getFreeIndexName(); - StringBuilder sb = new StringBuilder("CREATE "); - if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append("UNIQUE "); + boolean unique = Randomly.getBooleanWithRatherLowProbability(); + if (unique) { errors.add("Duplicate for key"); errors.add("Duplicate entry "); errors.add("A UNIQUE INDEX must include all columns in the table's partitioning function"); } - sb.append("INDEX "); - sb.append(indexName); + appendCreateIndex(unique); + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); sb.append(randomTable.getName()); sb.append("("); @@ -63,7 +68,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx errors.add("index already exist"); errors.add("Data truncation"); errors.add("key was too long"); - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java index ab03316cc..66c59a968 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -1,9 +1,7 @@ package sqlancer.yugabyte.ycql.gen; -import java.util.List; - import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; @@ -11,33 +9,31 @@ import sqlancer.yugabyte.ycql.YCQLToStringVisitor; import sqlancer.yugabyte.ycql.ast.YCQLExpression; -public final class YCQLIndexGenerator { +public class YCQLIndexGenerator extends AbstractIndexGenerator { + + private final YCQLGlobalState globalState; - private YCQLIndexGenerator() { + public YCQLIndexGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { + return new YCQLIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); sb.append(" ON "); YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); - sb.append("("); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - } - sb.append(")"); + appendIndexColumnList(table.getRandomNonEmptyColumnSubset(), false); if (Randomly.getBoolean()) { sb.append(" WHERE "); YCQLExpression expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) @@ -49,7 +45,6 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { errors.add("Invalid CQL Statement"); errors.add( "Invalid Table Definition. Transactions cannot be enabled in an index of a table without transactions enabled."); - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index 6077dcb1e..ac5b3242f 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ysql.YSQLErrors; @@ -17,19 +17,21 @@ import sqlancer.yugabyte.ysql.YSQLVisitor; import sqlancer.yugabyte.ysql.ast.YSQLExpression; -public final class YSQLIndexGenerator { +public class YSQLIndexGenerator extends AbstractIndexGenerator { - private YSQLIndexGenerator() { + private final YSQLGlobalState globalState; + + public YSQLIndexGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(YSQLGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - if (Randomly.getBoolean()) { - sb.append(" UNIQUE"); - } - sb.append(" INDEX "); + return new YSQLIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(Randomly.getBoolean()); YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized // views String indexName = getNewIndexName(randomTable); @@ -122,7 +124,6 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); YSQLErrors.addCommonExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } private static String getNewIndexName(YSQLTable randomTable) { From 037e9e7abb2c81179218d34e550507dd9330341e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Apr 2026 23:42:25 +0800 Subject: [PATCH 063/132] CI: cache Rust build and pinned tarball downloads Three caches that avoid repeated network work, keyed on pinned versions: - Swatinem/rust-cache for the DataFusion Rust build (cargo registry + target) - actions/cache for the CockroachDB tarball (cockroachdb + cockroachdb-qpg) - actions/cache for the Doris tarball Tarballs are cached as raw .tgz/.tar.gz files rather than extracted dirs, so runtime state from previous runs is not persisted. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee4186650..9df326bba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,9 +154,15 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true + - name: Cache CockroachDB tarball + uses: actions/cache@v4 + with: + path: cockroach-v24.2.0.linux-amd64.tgz + key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz + tar xzf cockroach-v24.2.0.linux-amd64.tgz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user @@ -180,9 +186,15 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true + - name: Cache CockroachDB tarball + uses: actions/cache@v4 + with: + path: cockroach-v24.2.0.linux-amd64.tgz + key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz + tar xzf cockroach-v24.2.0.linux-amd64.tgz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user @@ -228,6 +240,10 @@ jobs: with: toolchain: stable override: true + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + workspaces: src/sqlancer/datafusion/server/datafusion_server - name: Build DataFusion Server run: | cd src/sqlancer/datafusion/server/datafusion_server @@ -690,10 +706,15 @@ jobs: run: | sudo apt update sudo apt install mysql-client --assume-yes + - name: Cache Apache Doris tarball + uses: actions/cache@v4 + with: + path: apache-doris-2.1.4-bin-x64.tar.gz + key: apache-doris-2.1.4-bin-x64-tarball - name: Set up Apache Doris run: | sudo sysctl -w vm.max_map_count=2000000 - wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz + [ -f apache-doris-2.1.4-bin-x64.tar.gz ] || wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz tar zxf apache-doris-2.1.4-bin-x64.tar.gz mv apache-doris-2.1.4-bin-x64 apache-doris sudo swapoff -a From ee60e918d06f0f11105a9182a9852a6fa1d3209c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 10:20:56 +0800 Subject: [PATCH 064/132] CI: drop CockroachDB tarball cache (no measured benefit) Warm-cache rerun showed CockroachDB setup at 11s vs 10s baseline - binaries.cockroachdb.com is fast enough that cache restore offers no saving. Keep the Rust build (-90s) and Doris (-216s) caches. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9df326bba..e85c5c219 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,15 +154,9 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Cache CockroachDB tarball - uses: actions/cache@v4 - with: - path: cockroach-v24.2.0.linux-amd64.tgz - key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz - tar xzf cockroach-v24.2.0.linux-amd64.tgz + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user @@ -186,15 +180,9 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Cache CockroachDB tarball - uses: actions/cache@v4 - with: - path: cockroach-v24.2.0.linux-amd64.tgz - key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz - tar xzf cockroach-v24.2.0.linux-amd64.tgz + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user From b2df941f170bb87a975b3a55cc14692ef9b68514 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 00:07:52 +0800 Subject: [PATCH 065/132] Refactor: extract common CREATE TABLE logic into AbstractTableGenerator Introduce AbstractTableGenerator with appendCreateTable header helpers and appendColumnDefinitions / appendColumnDefinitionList iteration helpers that delegate to an overridable appendColumnDefinition(C) (default: "name type"). Convert four concrete table generators (QuestDB, HSQLDB, Databend, DuckDB) from plain classes to instance-based subclasses whose logic lives in buildStatement(). Public entry points (getQuery) are preserved so callers in the provider classes don't need to change. Generators with DB-specific shape (temporary/unlogged modifiers, in-place column building, partitioning, custom return types, etc.) are left alone. --- .../common/gen/AbstractTableGenerator.java | 42 +++++++++ .../databend/gen/DatabendTableGenerator.java | 65 +++++++------ .../duckdb/gen/DuckDBTableGenerator.java | 92 ++++++++++--------- .../hsqldb/gen/HSQLDBTableGenerator.java | 60 ++++++------ .../questdb/gen/QuestDBTableGenerator.java | 41 ++++----- 5 files changed, 178 insertions(+), 122 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractTableGenerator.java diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java new file mode 100644 index 000000000..e1cf34832 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -0,0 +1,42 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractTableGenerator> extends AbstractGenerator { + + protected void appendCreateTable(String tableName) { + appendCreateTable(tableName, false); + } + + protected void appendCreateTable(String tableName, boolean ifNotExists) { + sb.append("CREATE TABLE "); + if (ifNotExists) { + sb.append("IF NOT EXISTS "); + } + sb.append(tableName); + } + + protected void appendColumnDefinitions(List columns) { + sb.append("("); + appendColumnDefinitionList(columns); + sb.append(")"); + } + + protected void appendColumnDefinitionList(List columns) { + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + appendColumnDefinition(columns.get(i)); + } + } + + protected void appendColumnDefinition(C column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 514d740d0..2c3416538 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -4,8 +4,8 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.gen.TypedExpressionGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; @@ -15,44 +15,49 @@ import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendExpression; -public class DatabendTableGenerator { +public class DatabendTableGenerator extends AbstractTableGenerator { + + private DatabendGlobalState globalState; + private TypedExpressionGenerator gen; + + public DatabendTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { DatabendErrors.addExpressionErrors(errors); - StringBuilder sb = new StringBuilder(); String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName); List columns = getNewColumns(); - TypedExpressionGenerator gen = new DatabendNewExpressionGenerator( - globalState).setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); + gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + appendColumnDefinitions(columns); + } - if (globalState.getDbmsSpecificOptions().testNotNullConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" NOT NULL"); - } else { - sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 - } + @Override + protected void appendColumnDefinition(DatabendColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); - if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { - sb.append(" DEFAULT("); - sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 - gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); - sb.append(")"); - } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } else { + sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 } - sb.append(")"); - return new SQLQueryAdapter(sb.toString(), errors, true); + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 + gen.generateConstant(column.getType().getPrimitiveDataType()))); + sb.append(")"); + } } private static List getNewColumns() { diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index ea6d3537f..c8ac6f28d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -5,8 +5,8 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -16,50 +16,28 @@ import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBTableGenerator { +public class DuckDBTableGenerator extends AbstractTableGenerator { + + private DuckDBGlobalState globalState; + private UntypedExpressionGenerator gen; + + public DuckDBTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName); List columns = getNewColumns(); - UntypedExpressionGenerator gen = new DuckDBExpressionGenerator(globalState) - .setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() - && columns.get(i).getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { - sb.append(" COLLATE "); - sb.append(getRandomCollate()); - } - if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" UNIQUE"); - } - if (globalState.getDbmsSpecificOptions().testNotNullConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" NOT NULL"); - } - if (globalState.getDbmsSpecificOptions().testCheckConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" CHECK("); - sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); - DuckDBErrors.addExpressionErrors(errors); - sb.append(")"); - } - if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { - sb.append(" DEFAULT("); - sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); - sb.append(")"); - } - } + gen = new DuckDBExpressionGenerator(globalState).setColumns(columns); + sb.append("("); + appendColumnDefinitionList(columns); if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { errors.add("Invalid type for index"); List primaryKeyColumns = Randomly.nonEmptySubset(columns); @@ -68,7 +46,37 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(")"); } sb.append(")"); - return new SQLQueryAdapter(sb.toString(), errors, true); + } + + @Override + protected void appendColumnDefinition(DuckDBColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + && column.getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { + sb.append(" COLLATE "); + sb.append(getRandomCollate()); + } + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" UNIQUE"); + } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } + if (globalState.getDbmsSpecificOptions().testCheckConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" CHECK("); + sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); + DuckDBErrors.addExpressionErrors(errors); + sb.append(")"); + } + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + } } public static String getRandomCollate() { diff --git a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java index 48606e9bf..30249e780 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java @@ -5,44 +5,48 @@ import javax.annotation.Nullable; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; -public class HSQLDBTableGenerator { +public class HSQLDBTableGenerator extends AbstractTableGenerator { + + private HSQLDBProvider.HSQLDBGlobalState globalState; + private String tableName; + + public HSQLDBTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @Nullable String tableName) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + this.tableName = tableName; + return getStatement(); + } + + @Override + public void buildStatement() { String name = tableName; - if (tableName == null) { + if (name == null) { name = globalState.getSchema().getFreeTableName(); } - sb.append("CREATE TABLE "); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(name); - sb.append("("); - List columns = getNewColumns(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType().getType().name()); - if (columns.get(i).getType().getSize() > 0) { - // Cannot specify size for non composite data types - sb.append("("); - sb.append(columns.get(i).getType().getSize()); - sb.append(")"); - } - } - sb.append(")"); + appendCreateTable(name, Randomly.getBoolean()); + appendColumnDefinitions(getNewColumns()); sb.append(";"); - return new SQLQueryAdapter(sb.toString(), errors, true); + } + + @Override + protected void appendColumnDefinition(HSQLDBSchema.HSQLDBColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType().getType().name()); + if (column.getType().getSize() > 0) { + // Cannot specify size for non composite data types + sb.append("("); + sb.append(column.getType().getSize()); + sb.append(")"); + } } private static List getNewColumns() { diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java index a308b8d10..d4b17e76b 100644 --- a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -5,40 +5,37 @@ import javax.annotation.Nullable; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; -public class QuestDBTableGenerator { +public class QuestDBTableGenerator extends AbstractTableGenerator { + + private QuestDBGlobalState globalState; + private String tableName; + + public QuestDBTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(QuestDBGlobalState globalState, @Nullable String tableName) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + this.tableName = tableName; + return getStatement(); + } + + @Override + public void buildStatement() { String name = tableName; - if (tableName == null) { + if (name == null) { name = globalState.getSchema().getFreeTableName(); } - sb.append("CREATE TABLE "); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(name); - sb.append("("); - List columns = getNewColumns(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - } - sb.append(")"); + appendCreateTable(name, Randomly.getBoolean()); + appendColumnDefinitions(getNewColumns()); sb.append(";"); errors.add("table already exists"); - return new SQLQueryAdapter(sb.toString(), errors, true); } private static List getNewColumns() { From 8d53aabc5080095dc0350e452ed1bee8f04be149 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 00:24:33 +0800 Subject: [PATCH 066/132] Refactor: migrate Presto, YCQL, Spark table generators to AbstractTableGenerator Follow-up to the initial four migrations. Presto and YCQL fit the canonical CREATE TABLE [IF NOT EXISTS] (col type[, ...]) shape directly. Spark needs the columns pre-built before iteration so that appendColumnDefinitions can call back into the overridden appendColumnDefinition; this is safe because Spark's DEFAULT clause uses generateConstant, which does not reference other columns. Hive is left alone: its CHECK/DEFAULT constraints use generateExpression which reads the columnsToBeAdded list incrementally, so pre-building would change expression-generation behavior. --- .../presto/gen/PrestoTableGenerator.java | 70 +++++++------------ .../spark/gen/SparkTableGenerator.java | 45 +++++------- .../yugabyte/ycql/gen/YCQLTableGenerator.java | 37 +++++----- 3 files changed, 60 insertions(+), 92 deletions(-) diff --git a/src/sqlancer/presto/gen/PrestoTableGenerator.java b/src/sqlancer/presto/gen/PrestoTableGenerator.java index 1d7df2ee6..49e1346ad 100644 --- a/src/sqlancer/presto/gen/PrestoTableGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTableGenerator.java @@ -4,13 +4,35 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema.PrestoColumn; import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; -public class PrestoTableGenerator { +public class PrestoTableGenerator extends AbstractTableGenerator { + + private PrestoGlobalState globalState; + + public PrestoTableGenerator() { + this.canAffectSchema = true; + this.canonicalizeString = false; + } + + public SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { + String catalog = globalState.getDbmsSpecificOptions().catalog; + String schema = globalState.getDatabaseName(); + String tableName = globalState.getSchema().getFreeTableName(); + String qualifiedName = catalog + "." + schema + "." + tableName; + appendCreateTable(qualifiedName); + appendColumnDefinitions(getNewColumns()); + } private static List getNewColumns() { List columns = new ArrayList<>(); @@ -22,48 +44,4 @@ private static List getNewColumns() { return columns; } - public SQLQueryAdapter getQuery(PrestoGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - String catalog = globalState.getDbmsSpecificOptions().catalog; - String schema = globalState.getDatabaseName(); - - sb.append(catalog).append("."); - sb.append(schema).append("."); - - sb.append(tableName); - sb.append("("); - List columns = getNewColumns(); - // TypedExpressionGenerator, PrestoColumn, PrestoCompositeDataType> - // typedExpressionGenerator = new PrestoTypedExpressionGenerator(globalState).setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - PrestoColumn column = columns.get(i); - sb.append(column.getName()); - sb.append(" "); - sb.append(column.getType()); - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" UNIQUE"); - // } - // if (globalState.getDbmsSpecificOptions().testNotNullConstraints - // && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" NOT NULL"); - // } - } - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { - // errors.add("Invalid type for index"); - // List primaryKeyColumns = Randomly.nonEmptySubset(columns); - // sb.append(", PRIMARY KEY("); - // sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - // sb.append(")"); - // } - sb.append(")"); - - return new SQLQueryAdapter(sb.toString(), errors, true, false); - } - } diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java index 937e52248..2c26ea34c 100644 --- a/src/sqlancer/spark/gen/SparkTableGenerator.java +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.spark.SparkErrors; import sqlancer.spark.SparkGlobalState; @@ -15,7 +15,7 @@ import sqlancer.spark.SparkSchema.SparkTable; import sqlancer.spark.SparkToStringVisitor; -public class SparkTableGenerator { +public class SparkTableGenerator extends AbstractTableGenerator { private enum ColumnConstraints { NOT_NULL, DEFAULT @@ -27,7 +27,6 @@ private enum ColumnConstraints { private final SparkGlobalState globalState; private final String tableName; - private final StringBuilder sb = new StringBuilder(); private final SparkExpressionGenerator gen; private final SparkTable table; private final List columnsToBeAdded = new ArrayList<>(); @@ -37,28 +36,25 @@ public SparkTableGenerator(SparkGlobalState globalState, String tableName) { this.globalState = globalState; this.table = new SparkTable(tableName, columnsToBeAdded, false); this.gen = new SparkExpressionGenerator(globalState).setColumns(columnsToBeAdded); + this.canAffectSchema = true; + this.canonicalizeString = false; } public static SQLQueryAdapter generate(SparkGlobalState globalState, String tableName) { - SparkTableGenerator generator = new SparkTableGenerator(globalState, tableName); - return generator.create(); + return new SparkTableGenerator(globalState, tableName).getStatement(); } - private SQLQueryAdapter create() { - ExpectedErrors errors = new ExpectedErrors(); - - sb.append("CREATE TABLE "); - sb.append(globalState.getDatabaseName()); - sb.append("."); - sb.append(tableName); - sb.append(" ("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - if (i != 0) { - sb.append(", "); - } - appendColumn(i); + @Override + public void buildStatement() { + int columnCount = Randomly.smallNumber() + 1; + for (int i = 0; i < columnCount; i++) { + String columnName = DBMSCommon.createColumnName(i); + SparkDataType type = SparkSchema.SparkDataType.getRandomType(); + columnsToBeAdded.add(new SparkColumn(columnName, table, type)); } - sb.append(")"); + appendCreateTable(globalState.getDatabaseName() + "." + tableName); + sb.append(" "); + appendColumnDefinitions(columnsToBeAdded); sb.append(" USING PARQUET"); // TODO: implement PARTITION BY clause @@ -67,16 +63,13 @@ private SQLQueryAdapter create() { // TODO: randomly add some predefined TABLEPROPERTIES SparkErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true, false); } - private void appendColumn(int columnId) { - String columnName = DBMSCommon.createColumnName(columnId); - sb.append(columnName); + @Override + protected void appendColumnDefinition(SparkColumn column) { + sb.append(column.getName()); sb.append(" "); - SparkDataType randType = SparkSchema.SparkDataType.getRandomType(); - sb.append(randType); - columnsToBeAdded.add(new SparkColumn(columnName, table, randType)); + sb.append(column.getType()); appendColumnConstraint(); } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java index 148a1cdbc..c57f94287 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java @@ -5,35 +5,33 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLCompositeDataType; -public class YCQLTableGenerator { +public class YCQLTableGenerator extends AbstractTableGenerator { + + private YCQLGlobalState globalState; + + public YCQLTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName, Randomly.getBoolean()); List columns = getNewColumns(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - // todo PK, STATIC - } + sb.append("("); + appendColumnDefinitionList(columns); errors.add("Query timed out after PT2S"); errors.add("Invalid type for index"); List primaryKeyColumns = Randomly.nonEmptySubset(columns); @@ -41,7 +39,6 @@ public SQLQueryAdapter getQuery(YCQLGlobalState globalState) { sb.append(primaryKeyColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); sb.append(")"); sb.append(")"); - return new SQLQueryAdapter(sb.toString(), errors, true); } private static List getNewColumns() { From db0d9e62bc82abfa387272a969b20bf5061739a6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 00:31:33 +0800 Subject: [PATCH 067/132] Document AbstractTableGenerator helpers with example output Add short Javadoc to each helper showing what SQL fragment it emits, so subclass authors can pick the right entry point at a glance. --- .../common/gen/AbstractTableGenerator.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java index e1cf34832..c6ca55cbc 100644 --- a/src/sqlancer/common/gen/AbstractTableGenerator.java +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -6,10 +6,12 @@ public abstract class AbstractTableGenerator> extends AbstractGenerator { + /** Appends {@code CREATE TABLE }. */ protected void appendCreateTable(String tableName) { appendCreateTable(tableName, false); } + /** Appends {@code CREATE TABLE [IF NOT EXISTS ]}. */ protected void appendCreateTable(String tableName, boolean ifNotExists) { sb.append("CREATE TABLE "); if (ifNotExists) { @@ -18,12 +20,20 @@ protected void appendCreateTable(String tableName, boolean ifNotExists) { sb.append(tableName); } + /** + * Appends a parenthesized, comma-separated column definition list, e.g. {@code (c0 INT, c1 TEXT)}. Delegates each + * column's rendering to {@link #appendColumnDefinition(AbstractTableColumn)}. + */ protected void appendColumnDefinitions(List columns) { sb.append("("); appendColumnDefinitionList(columns); sb.append(")"); } + /** + * Appends a comma-separated column definition list without enclosing parentheses, e.g. {@code c0 INT, c1 TEXT}. + * Useful when subclasses also emit table-level constraints (e.g. {@code PRIMARY KEY (...)}) inside the same parens. + */ protected void appendColumnDefinitionList(List columns) { for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -33,6 +43,10 @@ protected void appendColumnDefinitionList(List columns) { } } + /** + * Appends a single column's definition. Default output is {@code }, e.g. {@code c0 INT}. Override to + * add constraints such as {@code NOT NULL}, {@code DEFAULT ...}, or {@code CHECK (...)}. + */ protected void appendColumnDefinition(C column) { sb.append(column.getName()); sb.append(" "); From 220835e5fb96383cf814ba9407e1891e282a528c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 25 Apr 2026 23:30:28 +0800 Subject: [PATCH 068/132] Fix checkstyle: add @param tags to AbstractTableGenerator Javadoc Co-Authored-By: Claude Opus 4.7 --- .../common/gen/AbstractTableGenerator.java | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java index c6ca55cbc..c8d1d9697 100644 --- a/src/sqlancer/common/gen/AbstractTableGenerator.java +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -6,12 +6,24 @@ public abstract class AbstractTableGenerator> extends AbstractGenerator { - /** Appends {@code CREATE TABLE }. */ + /** + * Appends {@code CREATE TABLE }. + * + * @param tableName + * the name of the table to create. + */ protected void appendCreateTable(String tableName) { appendCreateTable(tableName, false); } - /** Appends {@code CREATE TABLE [IF NOT EXISTS ]}. */ + /** + * Appends {@code CREATE TABLE [IF NOT EXISTS ]}. + * + * @param tableName + * the name of the table to create. + * @param ifNotExists + * whether to emit the {@code IF NOT EXISTS} clause. + */ protected void appendCreateTable(String tableName, boolean ifNotExists) { sb.append("CREATE TABLE "); if (ifNotExists) { @@ -23,6 +35,9 @@ protected void appendCreateTable(String tableName, boolean ifNotExists) { /** * Appends a parenthesized, comma-separated column definition list, e.g. {@code (c0 INT, c1 TEXT)}. Delegates each * column's rendering to {@link #appendColumnDefinition(AbstractTableColumn)}. + * + * @param columns + * the columns to render. */ protected void appendColumnDefinitions(List columns) { sb.append("("); @@ -33,6 +48,9 @@ protected void appendColumnDefinitions(List columns) { /** * Appends a comma-separated column definition list without enclosing parentheses, e.g. {@code c0 INT, c1 TEXT}. * Useful when subclasses also emit table-level constraints (e.g. {@code PRIMARY KEY (...)}) inside the same parens. + * + * @param columns + * the columns to render. */ protected void appendColumnDefinitionList(List columns) { for (int i = 0; i < columns.size(); i++) { @@ -46,6 +64,9 @@ protected void appendColumnDefinitionList(List columns) { /** * Appends a single column's definition. Default output is {@code }, e.g. {@code c0 INT}. Override to * add constraints such as {@code NOT NULL}, {@code DEFAULT ...}, or {@code CHECK (...)}. + * + * @param column + * the column whose definition to render. */ protected void appendColumnDefinition(C column) { sb.append(column.getName()); From 12048b67e9134820acc98037221fd1b41e1867c1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 26 Apr 2026 00:16:31 +0800 Subject: [PATCH 069/132] Refactor: extract common DELETE generation logic into AbstractDeleteGenerator Add appendDeleteFromTable, appendLimitClause, and appendReturningClause helpers in AbstractDeleteGenerator and migrate all 15 DELETE generators that extend it. Lift the WHERE-clause helper to AbstractGenerator since it is also used by UPDATE, partial-INDEX, and INSERT...ON CONFLICT generators, and migrate 17 of those generators to use it. Co-Authored-By: Claude Opus 4.7 --- .../gen/CockroachDBDeleteGenerator.java | 6 +-- .../gen/CockroachDBUpdateGenerator.java | 3 +- .../common/gen/AbstractDeleteGenerator.java | 50 +++++++++++++++++++ .../common/gen/AbstractGenerator.java | 13 +++++ .../databend/gen/DatabendDeleteGenerator.java | 6 +-- .../doris/gen/DorisDeleteGenerator.java | 6 +-- .../doris/gen/DorisUpdateGenerator.java | 3 +- .../duckdb/gen/DuckDBDeleteGenerator.java | 6 +-- src/sqlancer/h2/H2DeleteGenerator.java | 9 ++-- src/sqlancer/h2/H2UpdateGenerator.java | 3 +- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 3 +- .../mariadb/gen/MariaDBDeleteGenerator.java | 17 ++++--- .../gen/MaterializeDeleteGenerator.java | 7 +-- .../gen/MaterializeUpdateGenerator.java | 3 +- .../mysql/gen/MySQLDeleteGenerator.java | 3 +- .../mysql/gen/MySQLUpdateGenerator.java | 3 +- .../gen/OceanBaseDeleteGenerator.java | 3 +- .../gen/OceanBaseUpdateGenerator.java | 3 +- .../postgres/gen/PostgresDeleteGenerator.java | 13 ++--- .../postgres/gen/PostgresIndexGenerator.java | 3 +- .../postgres/gen/PostgresUpdateGenerator.java | 3 +- .../presto/gen/PrestoDeleteGenerator.java | 6 +-- .../presto/gen/PrestoIndexGenerator.java | 3 +- .../gen/dml/SQLite3DeleteGenerator.java | 6 +-- .../gen/dml/SQLite3UpdateGenerator.java | 3 +- .../tidb/gen/TiDBDeleteGenerator.java | 6 +-- .../tidb/gen/TiDBUpdateGenerator.java | 3 +- .../ycql/gen/YCQLDeleteGenerator.java | 6 +-- .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 3 +- .../ysql/gen/YSQLDeleteGenerator.java | 13 ++--- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 3 +- .../ysql/gen/YSQLUpdateGenerator.java | 3 +- 32 files changed, 116 insertions(+), 105 deletions(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index f0048d278..dd1e94aa9 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -24,12 +24,10 @@ public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { @Override public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); CockroachDBErrors.addExpressionErrors(errors); - sb.append(CockroachDBVisitor.asString(new CockroachDBExpressionGenerator(globalState) + appendWhereClause(CockroachDBVisitor.asString(new CockroachDBExpressionGenerator(globalState) .setColumns(table.getColumns()).generateExpression(CockroachDBDataType.BOOL.get()))); } else { errors.add("rejected: DELETE without WHERE clause (sql_safe_updates = true)"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index b367f5c59..06cc0f8ee 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -40,8 +40,7 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); + appendWhereClause(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); } errors.add("violates unique constraint"); errors.add("violates not-null constraint"); diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java index 8dfb0b0c6..f3b5a1955 100644 --- a/src/sqlancer/common/gen/AbstractDeleteGenerator.java +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -5,4 +5,54 @@ public abstract class AbstractDeleteGenerator extends AbstractGenerator { protected AbstractDeleteGenerator() { } + /** + * Appends {@code DELETE FROM }. + * + * @param tableName + * the name of the table to delete from. + */ + protected void appendDeleteFromTable(String tableName) { + appendDeleteFromTable(tableName, false); + } + + /** + * Appends {@code DELETE FROM [ONLY ]}. + * + * @param tableName + * the name of the table to delete from. + * @param only + * whether to emit the {@code ONLY} keyword (used by some databases to restrict deletion to the named + * table rather than its inheritance descendants). + */ + protected void appendDeleteFromTable(String tableName, boolean only) { + sb.append("DELETE FROM "); + if (only) { + sb.append("ONLY "); + } + sb.append(tableName); + } + + /** + * Appends {@code LIMIT } (with a leading space). + * + * @param value + * the LIMIT value, e.g. an integer literal or already-rendered expression. Converted via + * {@link StringBuilder#append(Object)}. + */ + protected void appendLimitClause(Object value) { + sb.append(" LIMIT "); + sb.append(value); + } + + /** + * Appends {@code RETURNING } (with a leading space). + * + * @param expression + * the rendered RETURNING expression. + */ + protected void appendReturningClause(String expression) { + sb.append(" RETURNING "); + sb.append(expression); + } + } diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java index 5d13fc746..dbdf100d5 100644 --- a/src/sqlancer/common/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -17,4 +17,17 @@ public SQLQueryAdapter getStatement() { public abstract void buildStatement(); + /** + * Appends {@code WHERE } (with a leading space). Subclasses are responsible for deciding whether to + * include the WHERE clause, typically based on a randomized boolean. Used by DELETE, UPDATE, partial-INDEX, and + * INSERT...ON CONFLICT generators. + * + * @param condition + * the rendered WHERE condition. + */ + protected void appendWhereClause(String condition) { + sb.append(" WHERE "); + sb.append(condition); + } + } diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index 5adabf175..22336fd72 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -22,11 +22,9 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); - sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + appendDeleteFromTable(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DatabendToStringVisitor.asString( + appendWhereClause(DatabendToStringVisitor.asString( new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN))); DatabendErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/doris/gen/DorisDeleteGenerator.java b/src/sqlancer/doris/gen/DorisDeleteGenerator.java index 66deaea8f..b155e0381 100644 --- a/src/sqlancer/doris/gen/DorisDeleteGenerator.java +++ b/src/sqlancer/doris/gen/DorisDeleteGenerator.java @@ -23,12 +23,10 @@ public static SQLQueryAdapter generate(DorisGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + appendWhereClause(DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) .setColumns(table.getColumns()).generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java index e0db2b7b4..93f835eff 100644 --- a/src/sqlancer/doris/gen/DorisUpdateGenerator.java +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -35,8 +35,7 @@ public void buildStatement() { sb.append(table.getName()); sb.append(" SET "); updateColumns(columns); - sb.append(" WHERE "); - sb.append(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + appendWhereClause(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addInsertErrors(errors); } diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index 42695a9f6..9f4f4ed6a 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -22,12 +22,10 @@ public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DuckDBToStringVisitor.asString( + appendWhereClause(DuckDBToStringVisitor.asString( new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } DuckDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java index dd0fdff34..291ee99c2 100644 --- a/src/sqlancer/h2/H2DeleteGenerator.java +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -20,17 +20,14 @@ public static SQLQueryAdapter getQuery(H2GlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(H2ToStringVisitor.asString( + appendWhereClause(H2ToStringVisitor.asString( new H2ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } if (Randomly.getBoolean()) { - sb.append(" LIMIT "); - sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); + appendLimitClause(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); } H2Errors.addExpressionErrors(errors); H2Errors.addDeleteErrors(errors); diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index fe63c7a4e..05e348038 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -34,8 +34,7 @@ public void buildStatement() { H2Errors.addInsertErrors(errors); H2Errors.addDeleteErrors(errors); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(H2ToStringVisitor.asString(gen.generateExpression())); + appendWhereClause(H2ToStringVisitor.asString(gen.generateExpression())); } H2Errors.addExpressionErrors(errors); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index 54380214f..2e6081df4 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -37,8 +37,7 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); - sb.append(HSQLDBToStringVisitor.asString( + appendWhereClause(HSQLDBToStringVisitor.asString( gen.generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)))); errors.add("data type of expression is not boolean"); HSQLDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index 2992f569d..da1651662 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -57,12 +57,13 @@ public void buildStatement() { sb.append(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); + String condition; if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); + condition = MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r)); } else { - sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); + condition = MariaDBVisitor.asString(expressionGenerator.getRandomExpression()); } + appendWhereClause(condition); } // ORDER BY + LIMIT @@ -75,18 +76,18 @@ public void buildStatement() { } if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" LIMIT "); - sb.append(Randomly.getNotCachedInteger(1, 10)); + appendLimitClause(Randomly.getNotCachedInteger(1, 10)); } // RETURNING clause (MariaDB >= 10.5) if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" RETURNING "); + String expression; if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); + expression = MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r)); } else { - sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); + expression = MariaDBVisitor.asString(expressionGenerator.getRandomExpression()); } + appendReturningClause(expression); } if (sb.toString().contains("RLIKE") || sb.toString().contains("REGEXP")) { diff --git a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java index 2aceddcc9..a5483ee1d 100644 --- a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java @@ -26,12 +26,9 @@ public void buildStatement() { errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - sb.append("DELETE FROM"); - sb.append(" "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, + appendWhereClause(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, table.getColumns(), MaterializeDataType.BOOLEAN))); } MaterializeCommon.addCommonExpressionErrors(errors); diff --git a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java index 7d338c027..abd288f99 100644 --- a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java @@ -52,10 +52,9 @@ public void buildStatement() { errors.add("but expression is of type"); MaterializeCommon.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); MaterializeExpression where = MaterializeExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), MaterializeDataType.BOOLEAN); - sb.append(MaterializeVisitor.asString(where)); + appendWhereClause(MaterializeVisitor.asString(where)); } } diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index 048ef335f..551264c4a 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -40,8 +40,7 @@ public void buildStatement() { sb.append(" FROM "); sb.append(randomTable.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(MySQLVisitor.asString(gen.generateExpression())); + appendWhereClause(MySQLVisitor.asString(gen.generateExpression())); MySQLErrors.addExpressionErrors(errors); } errors.addAll(Arrays.asList("doesn't have this option", diff --git a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java index 1aca84580..ad13a148a 100644 --- a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java @@ -34,9 +34,8 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); MySQLErrors.addExpressionErrors(errors); - sb.append(MySQLVisitor.asString(gen.generateExpression())); + appendWhereClause(MySQLVisitor.asString(gen.generateExpression())); } MySQLErrors.addInsertUpdateErrors(errors); errors.add("doesn't have this option"); diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index c46ace304..ec1aa7fc0 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -36,8 +36,7 @@ public void buildStatement() { sb.append(" FROM "); sb.append(randomTable.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + appendWhereClause(OceanBaseVisitor.asString(gen.generateExpression())); OceanBaseErrors.addExpressionErrors(errors); } errors.addAll(Arrays.asList("doesn't have this option", "Truncated incorrect DOUBLE value", diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index 44fdf39fe..51816e691 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -39,9 +39,8 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); OceanBaseErrors.addExpressionErrors(errors); - sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + appendWhereClause(OceanBaseVisitor.asString(gen.generateExpression())); errors.add("Data Too Long"); } errors.add("Duplicated primary key"); diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 452f5cdf4..3250b86c8 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -26,20 +26,13 @@ public void buildStatement() { errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - sb.append("DELETE FROM"); + appendDeleteFromTable(table.getName(), Randomly.getBoolean()); if (Randomly.getBoolean()) { - sb.append(" ONLY"); - } - sb.append(" "); - sb.append(table.getName()); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(PostgresVisitor.asString(PostgresExpressionGenerator.generateExpression(globalState, + appendWhereClause(PostgresVisitor.asString(PostgresExpressionGenerator.generateExpression(globalState, table.getColumns(), PostgresDataType.BOOLEAN))); } if (Randomly.getBoolean()) { - sb.append(" RETURNING "); - sb.append(PostgresVisitor + appendReturningClause(PostgresVisitor .asString(PostgresExpressionGenerator.generateExpression(globalState, table.getColumns()))); } PostgresCommon.addCommonExpressionErrors(errors); diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index bf70c32d4..c852684a9 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -107,10 +107,9 @@ public void buildStatement() { sb.append(")"); } if (Randomly.getBoolean()) { - sb.append(" WHERE "); PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(randomTable.getColumns()) .setGlobalState(globalState).generateExpression(PostgresDataType.BOOLEAN); - sb.append(PostgresVisitor.asString(expr)); + appendWhereClause(PostgresVisitor.asString(expr)); } errors.add("already contains data"); // CONCURRENT INDEX failed errors.add("You might need to add explicit type casts"); diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 1effad582..92257a95a 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -52,10 +52,9 @@ public void buildStatement() { errors.add("but expression is of type"); PostgresCommon.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); PostgresExpression where = PostgresExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), PostgresDataType.BOOLEAN); - sb.append(PostgresVisitor.asString(where)); + appendWhereClause(PostgresVisitor.asString(where)); } } diff --git a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java index 59b7b4174..4d5724992 100644 --- a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java +++ b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java @@ -24,12 +24,10 @@ public static SQLQueryAdapter generate(PrestoGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(PrestoToStringVisitor + appendWhereClause(PrestoToStringVisitor .asString(new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()))); } diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java index 5ec23c773..2d7b75cf8 100644 --- a/src/sqlancer/presto/gen/PrestoIndexGenerator.java +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -51,10 +51,9 @@ public void buildStatement() { } sb.append(")"); if (Randomly.getBoolean()) { - sb.append(" WHERE "); PrestoExpression expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); - sb.append(PrestoToStringVisitor.asString(expr)); + appendWhereClause(PrestoToStringVisitor.asString(expr)); } errors.add("already exists!"); } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 51b23eaa4..a19da484f 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -33,11 +33,9 @@ public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLi @Override public void buildStatement() { - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(SQLite3Visitor.asString( + appendWhereClause(SQLite3Visitor.asString( new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } SQLite3Errors.addExpectedExpressionErrors(errors); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 05f4b9a2e..9230f47ba 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -76,10 +76,9 @@ public void buildStatement() { } if (Randomly.getBoolean()) { - sb.append(" WHERE "); String whereClause = SQLite3Visitor.asString( new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression()); - sb.append(whereClause); + appendWhereClause(whereClause); } // ORDER BY and LIMIT are only supported by enabling a compile-time option diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index cbe5fc721..c83d1a6eb 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -41,8 +41,7 @@ public void buildStatement() { sb.append("FROM "); sb.append(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(TiDBVisitor.asString(gen.generateExpression())); + appendWhereClause(TiDBVisitor.asString(gen.generateExpression())); errors.add("Truncated incorrect"); errors.add("Data truncation"); errors.add("Truncated incorrect FLOAT value"); @@ -54,8 +53,7 @@ public void buildStatement() { .collect(Collectors.joining(", "))); } if (Randomly.getBoolean()) { - sb.append(" LIMIT "); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); + appendLimitClause(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); } errors.add("Bad Number"); errors.add("Truncated incorrect"); // https://github.com/pingcap/tidb/issues/24292 diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index af6430b48..dd79670d9 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -35,9 +35,8 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); - sb.append(TiDBVisitor.asString(gen.generateExpression())); + appendWhereClause(TiDBVisitor.asString(gen.generateExpression())); } TiDBErrors.addInsertErrors(errors); } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java index 6ebe0db5f..af99ae4a5 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java @@ -23,11 +23,9 @@ public static SQLQueryAdapter generate(YCQLGlobalState globalState) { @Override public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(YCQLToStringVisitor.asString( + appendWhereClause(YCQLToStringVisitor.asString( new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } YCQLErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java index 66c59a968..dd93b38d0 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -35,10 +35,9 @@ public void buildStatement() { sb.append(table.getName()); appendIndexColumnList(table.getRandomNonEmptyColumnSubset(), false); if (Randomly.getBoolean()) { - sb.append(" WHERE "); YCQLExpression expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(); - sb.append(YCQLToStringVisitor.asString(expr)); + appendWhereClause(YCQLToStringVisitor.asString(expr)); } errors.add("Query timed out after PT2S"); errors.add("Invalid SQL Statement"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java index 6e35c6862..9d3c7427d 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -27,20 +27,13 @@ public void buildStatement() { errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - sb.append("DELETE FROM"); + appendDeleteFromTable(table.getName(), Randomly.getBoolean()); if (Randomly.getBoolean()) { - sb.append(" ONLY"); - } - sb.append(" "); - sb.append(table.getName()); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(YSQLVisitor.asString( + appendWhereClause(YSQLVisitor.asString( YSQLExpressionGenerator.generateExpression(globalState, table.getColumns(), YSQLDataType.BOOLEAN))); } if (Randomly.getBoolean()) { - sb.append(" RETURNING "); - sb.append( + appendReturningClause( YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, table.getColumns()))); } YSQLErrors.addCommonExpressionErrors(errors); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index ac5b3242f..5453746b2 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -93,10 +93,9 @@ public void buildStatement() { sb.append(")"); } if (Randomly.getBoolean()) { - sb.append(" WHERE "); YSQLExpression expr = new YSQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()) .setGlobalState(globalState).generateExpression(YSQLDataType.BOOLEAN); - sb.append(YSQLVisitor.asString(expr)); + appendWhereClause(YSQLVisitor.asString(expr)); } errors.add("already contains data"); // CONCURRENT INDEX failed errors.add("You might need to add explicit type casts"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index 6c5fd4144..1a9e1de76 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -54,10 +54,9 @@ public void buildStatement() { errors.add("but expression is of type"); YSQLErrors.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); YSQLExpression where = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), YSQLDataType.BOOLEAN); - sb.append(YSQLVisitor.asString(where)); + appendWhereClause(YSQLVisitor.asString(where)); } } From 1d6722fe4c14db6cdb92b88c29a2486008a96079 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 10:13:07 +0800 Subject: [PATCH 070/132] Remove CnosDB support CnosDB returns EAGAIN ("Tskv: Index: index storage error: Resource temporarily unavailable (os error 11)") on DROP DATABASE under SQLancer's DDL load (cnosdb/cnosdb#2435). The CI job has been red on main for 14+ months, the only published image tags are rolling daily builds (no LTS), and upstream development appears stalled. Pin and retry attempts (#1341) did not help. Remove the CnosDB provider, tests, CI job, and documentation entries. Move CnosDB to the "Previously Supported DBMS" table in CONTRIBUTING.md. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/main.yml | 24 - CONTRIBUTING.md | 2 +- README.md | 2 +- src/check_names.py | 1 - src/sqlancer/Main.java | 2 - src/sqlancer/cnosdb/CnosDBBugs.java | 13 - .../cnosdb/CnosDBComparatorHelper.java | 145 ----- .../cnosdb/CnosDBCompoundDataType.java | 20 - src/sqlancer/cnosdb/CnosDBExpectedError.java | 87 --- src/sqlancer/cnosdb/CnosDBGlobalState.java | 28 - .../cnosdb/CnosDBLoggableFactory.java | 55 -- src/sqlancer/cnosdb/CnosDBOptions.java | 28 - src/sqlancer/cnosdb/CnosDBOracleFactory.java | 39 -- src/sqlancer/cnosdb/CnosDBProvider.java | 123 ----- src/sqlancer/cnosdb/CnosDBSchema.java | 243 -------- .../cnosdb/CnosDBToStringVisitor.java | 278 ---------- src/sqlancer/cnosdb/CnosDBVisitor.java | 102 ---- src/sqlancer/cnosdb/ast/CnosDBAggregate.java | 113 ---- src/sqlancer/cnosdb/ast/CnosDBAlias.java | 35 -- .../cnosdb/ast/CnosDBBetweenOperation.java | 34 -- .../ast/CnosDBBinaryArithmeticOperation.java | 69 --- .../ast/CnosDBBinaryComparisonOperation.java | 57 -- .../ast/CnosDBBinaryLogicalOperation.java | 33 -- .../cnosdb/ast/CnosDBCastOperation.java | 60 -- .../cnosdb/ast/CnosDBColumnValue.java | 27 - .../cnosdb/ast/CnosDBConcatOperation.java | 22 - src/sqlancer/cnosdb/ast/CnosDBConstant.java | 520 ------------------ src/sqlancer/cnosdb/ast/CnosDBExpression.java | 14 - src/sqlancer/cnosdb/ast/CnosDBFunction.java | 30 - .../ast/CnosDBFunctionWithUnknownResult.java | 104 ---- .../cnosdb/ast/CnosDBInOperation.java | 35 -- src/sqlancer/cnosdb/ast/CnosDBJoin.java | 46 -- .../cnosdb/ast/CnosDBLikeOperation.java | 22 - .../cnosdb/ast/CnosDBOrderByTerm.java | 37 -- .../cnosdb/ast/CnosDBPostfixOperation.java | 97 ---- .../cnosdb/ast/CnosDBPostfixText.java | 29 - .../cnosdb/ast/CnosDBPrefixOperation.java | 73 --- src/sqlancer/cnosdb/ast/CnosDBSelect.java | 102 ---- src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java | 28 - src/sqlancer/cnosdb/client/CnosDBClient.java | 110 ---- .../cnosdb/client/CnosDBConnection.java | 27 - .../cnosdb/client/CnosDBException.java | 9 - .../cnosdb/client/CnosDBResultSet.java | 52 -- src/sqlancer/cnosdb/gen/CnosDBCommon.java | 31 -- .../cnosdb/gen/CnosDBExpressionGenerator.java | 461 ---------------- .../cnosdb/gen/CnosDBInsertGenerator.java | 59 -- .../cnosdb/gen/CnosDBTableGenerator.java | 77 --- .../cnosdb/oracle/CnosDBNoRECBase.java | 23 - .../cnosdb/oracle/CnosDBNoRECOracle.java | 171 ------ .../oracle/tlp/CnosDBTLPAggregateOracle.java | 176 ------ .../cnosdb/oracle/tlp/CnosDBTLPBase.java | 112 ---- .../oracle/tlp/CnosDBTLPHavingOracle.java | 65 --- .../oracle/tlp/CnosDBTLPWhereOracle.java | 46 -- .../cnosdb/query/CnosDBOtherQuery.java | 32 -- .../cnosdb/query/CnosDBQueryAdapter.java | 42 -- .../cnosdb/query/CnosDBQueryProvider.java | 6 - .../cnosdb/query/CnosDBSelectQuery.java | 39 -- test/sqlancer/dbms/TestCnosDBNoREC.java | 22 - test/sqlancer/dbms/TestCnosDBTLP.java | 22 - test/sqlancer/dbms/TestConfig.java | 1 - 60 files changed, 2 insertions(+), 4360 deletions(-) delete mode 100644 src/sqlancer/cnosdb/CnosDBBugs.java delete mode 100644 src/sqlancer/cnosdb/CnosDBComparatorHelper.java delete mode 100644 src/sqlancer/cnosdb/CnosDBCompoundDataType.java delete mode 100644 src/sqlancer/cnosdb/CnosDBExpectedError.java delete mode 100644 src/sqlancer/cnosdb/CnosDBGlobalState.java delete mode 100644 src/sqlancer/cnosdb/CnosDBLoggableFactory.java delete mode 100644 src/sqlancer/cnosdb/CnosDBOptions.java delete mode 100644 src/sqlancer/cnosdb/CnosDBOracleFactory.java delete mode 100644 src/sqlancer/cnosdb/CnosDBProvider.java delete mode 100644 src/sqlancer/cnosdb/CnosDBSchema.java delete mode 100644 src/sqlancer/cnosdb/CnosDBToStringVisitor.java delete mode 100644 src/sqlancer/cnosdb/CnosDBVisitor.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBAggregate.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBAlias.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBCastOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBColumnValue.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBConstant.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBExpression.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunction.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBInOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBJoin.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixText.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBSelect.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBClient.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBConnection.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBException.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBResultSet.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBCommon.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java delete mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java delete mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBOtherQuery.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryProvider.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBSelectQuery.java delete mode 100644 test/sqlancer/dbms/TestCnosDBNoREC.java delete mode 100644 test/sqlancer/dbms/TestCnosDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e85c5c219..f0e1b429e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,30 +90,6 @@ jobs: - name: Run Tests run: CITUS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCitus test - cnosdb: - name: DBMS Tests (CnosDB, creation only) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - - name: Build SQLancer - run: mvn -B package -DskipTests=true - - name: Set up CnosDB - run: | - docker pull cnosdb/cnosdb:community-latest - docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done - - name: Run Tests - run: | - CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBNoREC test - sleep 20 - CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBTLP test - clickhouse: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea5baea1c..76b8833ea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,6 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | | Databend | Working | Typed | | | QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | -| CnosDB | Working | Typed | The implementation of CnosDB currently uses Restful API. | | Materialize | Working | Typed | | | Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | | Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | @@ -82,6 +81,7 @@ Some DBMS were once supported but subsequently removed. | Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | | MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | | StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. +| CnosDB | | This implementation was removed because the CnosDB image is unstable under SQLancer's DDL load (see [cnosdb/cnosdb#2435](https://github.com/cnosdb/cnosdb/issues/2435)) and the project appears no longer maintained. | ### Unfixed Bugs diff --git a/README.md b/README.md index 134f47666..f41e32d3c 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Usage: SQLancer [options] [command] [command options] **Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. -**Supported DBMSs.** SQLancer requires DBMS-specific code for each DBMS that it supports. As of January 2025, it provides support for Citus, ClickHouse, CnosDB, CockroachDB, Databend, (Apache) DataFusion, (Apache) Doris, DuckDB, H2, HSQLDB, MariaDB, Materialize, MySQL, OceanBase, PostgreSQL, Presto, QuestDB, SQLite3, TiDB, and YugabyteDB. The extent to which the individual DBMSs are supported [differs](https://github.com/sqlancer/sqlancer/blob/documentation-approaches/CONTRIBUTING.md). +**Supported DBMSs.** SQLancer requires DBMS-specific code for each DBMS that it supports. As of January 2025, it provides support for Citus, ClickHouse, CockroachDB, Databend, (Apache) DataFusion, (Apache) Doris, DuckDB, H2, HSQLDB, MariaDB, Materialize, MySQL, OceanBase, PostgreSQL, Presto, QuestDB, SQLite3, TiDB, and YugabyteDB. The extent to which the individual DBMSs are supported [differs](https://github.com/sqlancer/sqlancer/blob/documentation-approaches/CONTRIBUTING.md). # Approaches and Papers diff --git a/src/check_names.py b/src/check_names.py index f2ab346c6..453580f88 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -35,7 +35,6 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files: dict[str:List[str]] = dict() name_to_files["Citus"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "citus")) name_to_files["ClickHouse"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "clickhouse")) - name_to_files["CnosDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cnosdb")) name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) name_to_files["DataFusion"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "datafusion")) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index faf35e3c9..f273f5b95 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -26,7 +26,6 @@ import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.cnosdb.CnosDBProvider; import sqlancer.cockroachdb.CockroachDBProvider; import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; @@ -750,7 +749,6 @@ private static void checkForIssue799(List> providers) "No DBMS implementations (i.e., instantiations of the DatabaseProvider class) were found. You likely ran into an issue described in https://github.com/sqlancer/sqlancer/issues/799. As a workaround, I now statically load all supported providers as of June 7, 2023."); providers.add(new CitusProvider()); providers.add(new ClickHouseProvider()); - providers.add(new CnosDBProvider()); providers.add(new CockroachDBProvider()); providers.add(new DatabendProvider()); providers.add(new DorisProvider()); diff --git a/src/sqlancer/cnosdb/CnosDBBugs.java b/src/sqlancer/cnosdb/CnosDBBugs.java deleted file mode 100644 index 4e6eb96e9..000000000 --- a/src/sqlancer/cnosdb/CnosDBBugs.java +++ /dev/null @@ -1,13 +0,0 @@ -package sqlancer.cnosdb; - -public final class CnosDBBugs { - - // https://github.com/cnosdb/cnosdb/issues/786 - public static final boolean BUG786 = true; - - // https://github.com/apache/arrow-rs/issues/3547 - public static final boolean BUG3547 = true; - - private CnosDBBugs() { - } -} diff --git a/src/sqlancer/cnosdb/CnosDBComparatorHelper.java b/src/sqlancer/cnosdb/CnosDBComparatorHelper.java deleted file mode 100644 index 46b6ba615..000000000 --- a/src/sqlancer/cnosdb/CnosDBComparatorHelper.java +++ /dev/null @@ -1,145 +0,0 @@ -package sqlancer.cnosdb; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.function.UnaryOperator; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.cnosdb.query.CnosDBSelectQuery; -import sqlancer.common.query.ExpectedErrors; - -public final class CnosDBComparatorHelper { - - private CnosDBComparatorHelper() { - } - - public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, - CnosDBGlobalState state) throws Exception { - if (state.getOptions().logEachSelect()) { - // TODO: refactor me - state.getLogger().writeCurrent(queryString); - try { - state.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, errors); - List result = new ArrayList<>(); - CnosDBResultSet resultSet; - try { - q.executeAndGet(state); - resultSet = q.getResultSet(); - if (resultSet == null) { - throw new AssertionError(q); - } - while (resultSet.next()) { - result.add(resultSet.getString(1)); - } - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw e; - } - if (e instanceof NumberFormatException) { - throw new IgnoreMeException(); - } - if (e.getMessage() == null) { - throw new AssertionError(queryString, e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(queryString, e); - } - - return result; - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - String originalQueryString, List combinedString, CnosDBGlobalState state) { - if (resultSet.size() != secondResultSet.size()) { - String queryFormatString = "-- %s;\n-- cardinality: %d"; - String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); - String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), - secondResultSet.size()); - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", - resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); - throw new AssertionError(assertionMessage); - } - - Set firstHashSet = new HashSet<>(resultSet); - Set secondHashSet = new HashSet<>(secondResultSet); - - if (!firstHashSet.equals(secondHashSet)) { - Set firstResultSetMisses = new HashSet<>(firstHashSet); - firstResultSetMisses.removeAll(secondHashSet); - Set secondResultSetMisses = new HashSet<>(secondHashSet); - secondResultSetMisses.removeAll(firstHashSet); - String queryFormatString = "-- %s;\n-- misses: %s"; - String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); - String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), - secondResultSetMisses); - // update the SELECT queries to be logged at the bottom of the error log file - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", - firstQueryString, secondQueryString); - throw new AssertionError(assertionMessage); - } - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - String originalQueryString, List combinedString, CnosDBGlobalState state, - UnaryOperator canonicalizationRule) { - // Overloaded version of assumeResultSetsAreEqual that takes a canonicalization function which is applied to - // both result sets before their comparison. - List canonicalizedResultSet = resultSet.stream().map(canonicalizationRule).collect(Collectors.toList()); - List canonicalizedSecondResultSet = secondResultSet.stream().map(canonicalizationRule) - .collect(Collectors.toList()); - assumeResultSetsAreEqual(canonicalizedResultSet, canonicalizedSecondResultSet, originalQueryString, - combinedString, state); - } - - public static List getCombinedResultSet(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, - ExpectedErrors errors) throws Exception { - List secondResultSet; - if (asUnion) { - String unionString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL " - + thirdQueryString; - combinedString.add(unionString); - secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); - } else { - secondResultSet = new ArrayList<>(); - secondResultSet.addAll(getResultSetFirstColumnAsString(firstQueryString, errors, state)); - secondResultSet.addAll(getResultSetFirstColumnAsString(secondQueryString, errors, state)); - secondResultSet.addAll(getResultSetFirstColumnAsString(thirdQueryString, errors, state)); - combinedString.add(firstQueryString); - combinedString.add(secondQueryString); - combinedString.add(thirdQueryString); - } - return secondResultSet; - } - - public static List getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, - ExpectedErrors errors) throws Exception { - String unionString; - if (asUnion) { - unionString = firstQueryString + " UNION " + secondQueryString + " UNION " + thirdQueryString; - } else { - unionString = "SELECT DISTINCT * FROM (" + firstQueryString + " UNION ALL " + secondQueryString - + " UNION ALL " + thirdQueryString + ")"; - } - List secondResultSet; - combinedString.add(unionString); - secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); - return secondResultSet; - } -} diff --git a/src/sqlancer/cnosdb/CnosDBCompoundDataType.java b/src/sqlancer/cnosdb/CnosDBCompoundDataType.java deleted file mode 100644 index 034f0fc90..000000000 --- a/src/sqlancer/cnosdb/CnosDBCompoundDataType.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public final class CnosDBCompoundDataType { - - private final CnosDBDataType dataType; - - private CnosDBCompoundDataType(CnosDBDataType dataType) { - this.dataType = dataType; - } - - public static CnosDBCompoundDataType create(CnosDBDataType type) { - return new CnosDBCompoundDataType(type); - } - - public CnosDBDataType getDataType() { - return dataType; - } -} diff --git a/src/sqlancer/cnosdb/CnosDBExpectedError.java b/src/sqlancer/cnosdb/CnosDBExpectedError.java deleted file mode 100644 index 61dba101b..000000000 --- a/src/sqlancer/cnosdb/CnosDBExpectedError.java +++ /dev/null @@ -1,87 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.common.query.ExpectedErrors; - -public final class CnosDBExpectedError { - - private CnosDBExpectedError() { - } - - public static List getExpectedErrors() { - ArrayList errors = new ArrayList<>(); - - errors.add("have the same name. Consider aliasing"); - errors.add( - "error: Optimizer rule 'projection_push_down' failed due to unexpected error: Schema error: Schema contains duplicate qualified field name"); - errors.add("Projection references non-aggregate values:"); - errors.add("External err: Schema error: No field named"); - errors.add( - "Optimizer rule 'common_sub_expression_eliminate' failed due to unexpected error: Schema error: No field named"); - errors.add("Binary"); - errors.add("Invalid pattern in LIKE expression"); - errors.add("If the projection contains the time column, it must contain the field column."); - errors.add("Schema error: No field named"); - errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:"); - errors.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error"); - errors.add("Schema error: No field named "); - errors.add("err: External err: Schema error: No field named"); - errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error"); - errors.add("Csv error: CSV Writer does not support List"); - errors.add("This feature is not implemented: cross join."); - errors.add("Execution error: field position must be greater than zero"); - errors.add("First argument of `DATE_PART` must be non-null scalar Utf8"); - errors.add("Cannot create filter with non-boolean predicate 'NULL' returning Null"); - errors.add("requested character too large for encoding."); - errors.add("Can not find compatible types to compare Boolean with [Utf8]."); - errors.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT"); - errors.add("HAVING clause references non-aggregate values:"); - errors.add("Cannot create filter with non-boolean predicate"); - errors.add("negative substring length not allowed"); - errors.add("The function Sum does not support inputs of type Boolean."); - errors.add("The function Avg does not support inputs of type Boolean."); - errors.add("Percentile value must be between 0.0 and 1.0 inclusive"); - errors.add("Date part '' not supported"); - errors.add("Min/Max accumulator not implemented for type Boolean."); - errors.add("meta need get_series_id_by_filter"); - errors.add("Arrow: Cast error:"); - errors.add("Arrow error: Cast error:"); - errors.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:"); - errors.add("Arrow error: Divide by zero error"); - errors.add("desired percentile argument must be float literal"); - errors.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)"); - errors.add("Execution error: Date part"); - errors.add("Physical plan does not support logical expression MIN(Boolean"); - errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64"); - errors.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64."); - errors.add("Data type UInt64 not supported for binary operation '#' on dyn arrays."); - errors.add("Arrow: Divide by zero error"); - errors.add("The function ApproxPercentileCont does not support inputs of type Null."); - errors.add("can't be evaluated because there isn't a common type to coerce the types to"); - errors.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug"); - errors.add("The function ApproxMedian does not support inputs of type Null."); - errors.add("null character not permitted."); - errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Null."); - errors.add("This feature is not implemented"); - errors.add("The function Avg does not support inputs of type Null."); - errors.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]"); - errors.add( - "Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed."); - errors.add("Coercion from"); - - errors.add("Error parsing timestamp"); - errors.add("lpad requested length"); - errors.add("rpad requested length"); - errors.add("No function matches the given name and argument types"); - return errors; - } - - public static ExpectedErrors expectedErrors() { - ExpectedErrors res = new ExpectedErrors(); - res.addAll(getExpectedErrors()); - return res; - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBGlobalState.java b/src/sqlancer/cnosdb/CnosDBGlobalState.java deleted file mode 100644 index 9f34e03a5..000000000 --- a/src/sqlancer/cnosdb/CnosDBGlobalState.java +++ /dev/null @@ -1,28 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.ExecutionTimer; -import sqlancer.GlobalState; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.query.Query; - -public class CnosDBGlobalState extends GlobalState { - - @Override - protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { - boolean logExecutionTime = getOptions().logExecutionTime(); - if (success && getOptions().printSucceedingStatements()) { - System.out.println(q.getQueryString()); - } - if (logExecutionTime) { - getLogger().writeCurrent(" -- " + timer.end().asString()); - } - if (q.couldAffectSchema()) { - updateSchema(); - } - } - - @Override - public CnosDBSchema readSchema() throws Exception { - return CnosDBSchema.fromConnection(getConnection()); - } -} diff --git a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java deleted file mode 100644 index 407621c8b..000000000 --- a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java +++ /dev/null @@ -1,55 +0,0 @@ -package sqlancer.cnosdb; - -import java.io.PrintWriter; -import java.io.StringWriter; - -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.cnosdb.query.CnosDBQueryAdapter; -import sqlancer.common.log.Loggable; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.log.LoggedString; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; - -public class CnosDBLoggableFactory extends LoggableFactory { - - @Override - protected Loggable createLoggable(String input, String suffix) { - String completeString = input; - if (!input.endsWith(";")) { - completeString += ";"; - } - if (suffix != null && !suffix.isEmpty()) { - completeString += suffix; - } - return new LoggedString(completeString); - } - - @Override - public CnosDBQueryAdapter getQueryForStateToReproduce(String queryString) { - return new CnosDBOtherQuery(queryString, CnosDBExpectedError.expectedErrors()); - } - - @Override - public CnosDBQueryAdapter commentOutQuery(Query query) { - String queryString = query.getLogString(); - String newQueryString = "-- " + queryString; - ExpectedErrors errors = new ExpectedErrors(); - return new CnosDBOtherQuery(newQueryString, errors); - } - - @Override - protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { - String sb = "-- Time: " + time + "\n" + "-- Database: " + databaseName + "\n" + "-- Database version: " - + databaseVersion + "\n" + "-- seed value: " + seedValue + "\n"; - return new LoggedString(sb); - } - - @Override - public Loggable convertStacktraceToLoggable(Throwable throwable) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - throwable.printStackTrace(pw); - return new LoggedString("--" + sw.toString().replace("\n", "\n--")); - } -} diff --git a/src/sqlancer/cnosdb/CnosDBOptions.java b/src/sqlancer/cnosdb/CnosDBOptions.java deleted file mode 100644 index f101c2d38..000000000 --- a/src/sqlancer/cnosdb/CnosDBOptions.java +++ /dev/null @@ -1,28 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; - -@Parameters(separators = "=", commandDescription = "CnosDB (default port: " + CnosDBOptions.DEFAULT_PORT - + ", default host: " + CnosDBOptions.DEFAULT_HOST + ")") -public class CnosDBOptions implements DBMSSpecificOptions { - - public static final String DEFAULT_HOST = "localhost"; - public static final int DEFAULT_PORT = 31001; - - @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for CnosDB") - public List oracle = List.of(CnosDBOracleFactory.QUERY_PARTITIONING); - - @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the CnosDB", arity = 1) - public String connectionURL = String.format("http://%s:%d", CnosDBOptions.DEFAULT_HOST, CnosDBOptions.DEFAULT_PORT); - - @Override - public List getTestOracleFactory() { - return oracle; - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBOracleFactory.java b/src/sqlancer/cnosdb/CnosDBOracleFactory.java deleted file mode 100644 index 7cb9c4fc6..000000000 --- a/src/sqlancer/cnosdb/CnosDBOracleFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.OracleFactory; -import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPAggregateOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPHavingOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPWhereOracle; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; - -public enum CnosDBOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBNoRECOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - List> oracles = new ArrayList<>(); - oracles.add(new CnosDBTLPWhereOracle(globalState)); - oracles.add(new CnosDBTLPHavingOracle(globalState)); - oracles.add(new CnosDBTLPAggregateOracle(globalState)); - return new CompositeTestOracle<>(oracles, globalState); - } - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBProvider.java b/src/sqlancer/cnosdb/CnosDBProvider.java deleted file mode 100644 index 8b69c53b3..000000000 --- a/src/sqlancer/cnosdb/CnosDBProvider.java +++ /dev/null @@ -1,123 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.Objects; - -import com.google.auto.service.AutoService; - -import sqlancer.AbstractAction; -import sqlancer.DatabaseProvider; -import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; -import sqlancer.Randomly; -import sqlancer.StatementExecutor; -import sqlancer.cnosdb.client.CnosDBClient; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.cnosdb.gen.CnosDBInsertGenerator; -import sqlancer.cnosdb.gen.CnosDBTableGenerator; -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.cnosdb.query.CnosDBQueryProvider; -import sqlancer.common.log.LoggableFactory; - -@AutoService(DatabaseProvider.class) -public class CnosDBProvider extends ProviderAdapter { - - protected String username; - protected String password; - protected String host; - protected int port; - protected String databaseName; - - public CnosDBProvider() { - super(CnosDBGlobalState.class, CnosDBOptions.class); - } - - protected CnosDBProvider(Class globalClass, Class optionClass) { - super(globalClass, optionClass); - } - - protected static int mapActions(CnosDBGlobalState globalState, Action a) { - Randomly r = globalState.getRandomly(); - int nrPerformed; - if (Objects.requireNonNull(a) == Action.INSERT) { - nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - } else { - throw new AssertionError(a); - } - return nrPerformed; - - } - - @Override - protected void checkViewsAreValid(CnosDBGlobalState globalState) { - } - - @Override - public void generateDatabase(CnosDBGlobalState globalState) throws Exception { - createTables(globalState, Randomly.fromOptions(4, 5, 6)); - prepareTables(globalState); - - } - - @Override - public CnosDBConnection createDatabase(CnosDBGlobalState globalState) throws Exception { - - username = globalState.getOptions().getUserName(); - password = globalState.getOptions().getPassword(); - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); - databaseName = globalState.getDatabaseName(); - CnosDBClient client = new CnosDBClient(host, port, username, password, databaseName); - CnosDBConnection connection = new CnosDBConnection(client); - client.execute("DROP DATABASE IF EXISTS " + databaseName); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - client.execute("CREATE DATABASE " + databaseName); - globalState.getState().logStatement("CREATE DATABASE " + databaseName); - - return connection; - } - - protected void createTables(CnosDBGlobalState globalState, int numTables) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < numTables) { - String tableName = String.format("m%d", globalState.getSchema().getDatabaseTables().size()); - CnosDBOtherQuery createTable = CnosDBTableGenerator.generate(tableName); - globalState.executeStatement(createTable); - } - } - - protected void prepareTables(CnosDBGlobalState globalState) throws Exception { - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - CnosDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public String getDBMSName() { - return "CnosDB".toLowerCase(); - } - - @Override - public LoggableFactory getLoggableFactory() { - return new CnosDBLoggableFactory(); - } - - public enum Action implements AbstractAction { - INSERT(CnosDBInsertGenerator::insert); - - private final CnosDBQueryProvider sqlQueryProvider; - - Action(CnosDBQueryProvider sqlQueryProvider) { - this.sqlQueryProvider = sqlQueryProvider; - } - - @Override - public CnosDBOtherQuery getQuery(CnosDBGlobalState state) throws Exception { - return new CnosDBOtherQuery(sqlQueryProvider.getQuery(state).getQueryString(), - CnosDBExpectedError.expectedErrors()); - } - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBSchema.java b/src/sqlancer/cnosdb/CnosDBSchema.java deleted file mode 100644 index 022969ce5..000000000 --- a/src/sqlancer/cnosdb/CnosDBSchema.java +++ /dev/null @@ -1,243 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import sqlancer.Randomly; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.common.schema.AbstractRowValue; -import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.common.schema.AbstractTables; -import sqlancer.common.schema.TableIndex; - -public class CnosDBSchema extends AbstractSchema { - - private final String databaseName; - - public CnosDBSchema(List databaseTables, String databaseName) { - super(databaseTables); - this.databaseName = databaseName; - } - - public static CnosDBDataType getColumnType(String typeString) { - switch (typeString.toLowerCase()) { - case "bigint": - return CnosDBDataType.INT; - case "boolean": - return CnosDBDataType.BOOLEAN; - case "string": - return CnosDBDataType.STRING; - case "double": - return CnosDBDataType.DOUBLE; - case "bigint unsigned": - case "unsigned": - return CnosDBDataType.UINT; - case "timestamp(nanosecond)": - return CnosDBDataType.TIMESTAMP; - default: - throw new AssertionError(typeString); - } - } - - public static CnosDBSchema fromConnection(CnosDBConnection con) throws Exception { - CnosDBResultSet tablesRes = con.getClient().executeQuery("SHOW TABLES"); - - List tables = new ArrayList<>(); - while (tablesRes.next()) { - String tableName = tablesRes.getString(1); - List columns = getTableColumns(con, tableName); - tables.add(new CnosDBTable(tableName, columns)); - } - - return new CnosDBSchema(tables, con.getClient().getDatabase()); - } - - protected static List getTableColumns(CnosDBConnection con, String tableName) throws Exception { - CnosDBResultSet columnsRes = con.getClient().executeQuery("DESCRIBE TABLE " + tableName); - List columns = new ArrayList<>(); - CnosDBTable table = new CnosDBTable(tableName, columns); - while (columnsRes.next()) { - String columnName = columnsRes.getString(1); - String columnType = columnsRes.getString(3).toLowerCase(); - CnosDBDataType dataType = CnosDBSchema.getColumnType(columnsRes.getString(2)); - CnosDBColumn column; - if (columnType.contentEquals("time")) { - column = new CnosDBTimeColumn(); - } else if (columnType.contentEquals("tag")) { - column = new CnosDBTagColumn(columnName); - } else { - column = new CnosDBFieldColumn(columnName, dataType); - } - column.setTable(table); - columns.add(column); - } - - return columns; - } - - public CnosDBTables getRandomTableNonEmptyTables() { - return new CnosDBTables(Randomly.nonEmptySubset(getDatabaseTables())); - } - - public String getDatabaseName() { - return databaseName; - } - - public enum CnosDBDataType { - INT, BOOLEAN, STRING, DOUBLE, UINT, TIMESTAMP; - - public static CnosDBDataType getRandomType() { - return Randomly.fromOptions(values()); - } - - public static CnosDBDataType getRandomTypeWithoutTimeStamp() { - List dataTypes = new ArrayList<>(Arrays.asList(values())); - dataTypes.remove(TIMESTAMP); - return Randomly.fromList(dataTypes); - } - } - - public static class CnosDBColumn extends AbstractTableColumn { - - public CnosDBColumn(String name, CnosDBDataType columnType) { - super(name, null, columnType); - } - - public static CnosDBColumn createDummy(String name) { - return new CnosDBColumn(name, CnosDBDataType.INT); - } - - } - - public static class CnosDBTagColumn extends CnosDBColumn { - public CnosDBTagColumn(String name) { - super(name, CnosDBDataType.STRING); - } - } - - public static class CnosDBTimeColumn extends CnosDBColumn { - public CnosDBTimeColumn() { - super("TIME", CnosDBDataType.TIMESTAMP); - } - } - - public static class CnosDBFieldColumn extends CnosDBColumn { - public CnosDBFieldColumn(String name, CnosDBDataType columnType) { - super(name, columnType); - assert columnType != CnosDBDataType.TIMESTAMP; - } - } - - public static class CnosDBTables extends AbstractTables { - - public CnosDBTables(List tables) { - super(tables); - } - - public CnosDBRowValue getRandomRowValue(CnosDBConnection con) { - return null; - } - - public List getRandomColumnsWithOnlyOneField() { - ArrayList res = new ArrayList<>(); - this.getTables().forEach(table -> res.addAll(table.getRandomColumnsWithOnlyOneField())); - return res; - } - - } - - public static class CnosDBRowValue extends AbstractRowValue { - - protected CnosDBRowValue(CnosDBTables tables, Map values) { - super(tables, values); - } - - } - - public static class CnosDBTable extends AbstractTable { - - public CnosDBTable(String tableName, List columns) { - super(tableName, columns, null, false); - } - - @Override - public List getColumns() { - List res = super.getColumns(); - boolean hasTime = false; - for (CnosDBColumn column : res) { - if (column instanceof CnosDBTimeColumn) { - hasTime = true; - break; - } - } - assert hasTime; - - return res; - } - - public List getRandomColumnsWithOnlyOneField() { - ArrayList res = new ArrayList<>(); - boolean hasField = false; - for (CnosDBColumn column : getColumns()) { - if (column instanceof CnosDBTagColumn && Randomly.getBoolean()) { - res.add(column); - } else if (column instanceof CnosDBFieldColumn && !hasField) { - res.add(column); - hasField = true; - } - } - return res; - } - - // SELECT COUNT(*) FROM table; - @Override - public long getNrRows(CnosDBGlobalState globalState) { - long res; - try { - CnosDBResultSet tableCountRes = globalState.getConnection().getClient() - .executeQuery("SELECT COUNT(time) FROM " + this.name); - tableCountRes.next(); - res = tableCountRes.getLong(1); - } catch (Exception e) { - res = 0; - } - return res; - } - - @Override - public List getRandomNonEmptyColumnSubset() { - List selectedColumns = new ArrayList<>(); - ArrayList remainingColumns = new ArrayList<>(this.getColumns()); - - remainingColumns.removeIf(column -> column instanceof CnosDBTimeColumn); - CnosDBTimeColumn timeColumn = new CnosDBTimeColumn(); - timeColumn.setTable(this); - selectedColumns.add(timeColumn); - - remainingColumns.stream().filter(column -> column instanceof CnosDBTagColumn).findFirst().ifPresent(tag -> { - selectedColumns.add(tag); - remainingColumns.remove(tag); - }); - - remainingColumns.stream().filter(column -> column instanceof CnosDBFieldColumn).findFirst() - .ifPresent(field -> { - selectedColumns.add(field); - remainingColumns.remove(field); - }); - - int nr = Math.min(Randomly.smallNumber() + 1, remainingColumns.size()); - for (int i = 0; i < nr; i++) { - selectedColumns - .add(remainingColumns.remove((int) Randomly.getNotCachedInteger(0, remainingColumns.size()))); - } - return selectedColumns; - } - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java deleted file mode 100644 index 388e2ccd8..000000000 --- a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java +++ /dev/null @@ -1,278 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.Randomly; -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBBetweenOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBFunction; -import sqlancer.cnosdb.ast.CnosDBInOperation; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBLikeOperation; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixText; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.ast.CnosDBSimilarTo; -import sqlancer.common.visitor.BinaryOperation; -import sqlancer.common.visitor.ToStringVisitor; - -public final class CnosDBToStringVisitor extends ToStringVisitor implements CnosDBVisitor { - - @Override - public void visitSpecific(CnosDBExpression expr) { - CnosDBVisitor.super.visit(expr); - } - - @Override - public void visit(CnosDBConstant constant) { - sb.append(constant.getTextRepresentation()); - } - - @Override - public String get() { - return sb.toString(); - } - - @Override - public void visit(CnosDBPostfixOperation op) { - sb.append("("); - visit(op.getExpression()); - sb.append(")"); - sb.append(" "); - sb.append(op.getOperatorTextRepresentation()); - } - - @Override - public void visit(CnosDBColumnValue c) { - sb.append(c.getColumn().getFullQualifiedName()); - } - - @Override - public void visit(CnosDBPrefixOperation op) { - sb.append(op.getTextRepresentation()); - sb.append(" ("); - visit(op.getExpression()); - sb.append(")"); - } - - @Override - public void visit(CnosDBFromTable from) { - sb.append(from.getTable().getName()); - } - - @Override - public void visit(CnosDBSubquery subquery) { - sb.append("("); - visit(subquery.getSelect()); - sb.append(") AS "); - sb.append(subquery.getName()); - } - - @Override - public void visit(CnosDBSelect s) { - sb.append("SELECT "); - switch (s.getSelectOption()) { - case DISTINCT: - sb.append("DISTINCT "); - if (s.getDistinctOnClause() != null) { - sb.append("ON ("); - visit(s.getDistinctOnClause()); - sb.append(") "); - } - break; - case ALL: - sb.append(Randomly.fromOptions("ALL ", "")); - break; - default: - throw new AssertionError(); - } - if (s.getFetchColumns() == null) { - sb.append("*"); - } else { - visit(s.getFetchColumns()); - } - sb.append(" FROM "); - visit(s.getFromList()); - - for (CnosDBJoin j : s.getJoinClauses()) { - sb.append(" "); - switch (j.getType()) { - case INNER: - if (Randomly.getBoolean()) { - sb.append("INNER "); - } - sb.append("JOIN"); - break; - case LEFT: - sb.append("LEFT OUTER JOIN"); - break; - case RIGHT: - sb.append("RIGHT OUTER JOIN"); - break; - case FULL: - sb.append("FULL OUTER JOIN"); - break; - // case CROSS: - // sb.append("CROSS JOIN"); - // break; - default: - throw new AssertionError(j.getType()); - } - sb.append(" "); - visit(j.getTableReference()); - // if (j.getType() != CnosDBJoinType.CROSS) { - sb.append(" ON "); - visit(j.getOnClause()); - // } - } - - if (s.getWhereClause() != null) { - sb.append(" WHERE "); - visit(s.getWhereClause()); - } - if (!s.getGroupByExpressions().isEmpty()) { - sb.append(" GROUP BY "); - visit(s.getGroupByExpressions()); - } - if (s.getHavingClause() != null) { - sb.append(" HAVING "); - visit(s.getHavingClause()); - - } - if (!s.getOrderByClauses().isEmpty()) { - sb.append(" ORDER BY "); - visit(s.getOrderByClauses()); - } - if (s.getLimitClause() != null) { - sb.append(" LIMIT "); - visit(s.getLimitClause()); - } - - if (s.getOffsetClause() != null) { - sb.append(" OFFSET "); - visit(s.getOffsetClause()); - } - } - - @Override - public void visit(CnosDBOrderByTerm op) { - visit(op.getExpr()); - sb.append(" "); - sb.append(op.getOrder()); - } - - @Override - public void visit(CnosDBFunction f) { - sb.append(f.getFunctionName()); - sb.append("("); - int i = 0; - for (CnosDBExpression arg : f.getArguments()) { - if (i++ != 0) { - sb.append(", "); - } - visit(arg); - } - sb.append(")"); - } - - @Override - public void visit(CnosDBCastOperation cast) { - sb.append("CAST( "); - visit(cast.getExpression()); - sb.append(" AS "); - appendType(cast); - sb.append(")"); - } - - private void appendType(CnosDBCastOperation cast) { - CnosDBCompoundDataType compoundType = cast.getCompoundType(); - switch (compoundType.getDataType()) { - case BOOLEAN: - sb.append("BOOLEAN"); - break; - case INT: - sb.append("BIGINT"); - break; - case STRING: - sb.append(Randomly.fromOptions("STRING")); - break; - case DOUBLE: - sb.append("DOUBLE"); - break; - case UINT: - sb.append("BIGINT UNSIGNED"); - break; - case TIMESTAMP: - sb.append("TIMESTAMP"); - break; - - default: - throw new AssertionError(cast.getType()); - } - } - - @Override - public void visit(CnosDBBetweenOperation op) { - sb.append("("); - visit(op.getExpr()); - sb.append(") BETWEEN ("); - visit(op.getLeft()); - sb.append(") AND ("); - visit(op.getRight()); - sb.append(")"); - } - - @Override - public void visit(CnosDBInOperation op) { - sb.append("("); - visit(op.getExpr()); - sb.append(")"); - if (!op.isTrue()) { - sb.append(" NOT"); - } - sb.append(" IN ("); - visit(op.getListElements()); - sb.append(")"); - } - - @Override - public void visit(CnosDBPostfixText op) { - visit(op.getExpr()); - sb.append(op.getText()); - } - - @Override - public void visit(CnosDBAggregate op) { - sb.append(op.getFunction()); - sb.append("("); - visit(op.getArgs()); - sb.append(")"); - } - - @Override - public void visit(CnosDBSimilarTo op) { - sb.append("("); - visit(op.getString()); - sb.append(" SIMILAR TO "); - visit(op.getSimilarTo()); - sb.append(")"); - } - - @Override - public void visit(CnosDBBinaryLogicalOperation op) { - super.visit((BinaryOperation) op); - } - - @Override - public void visit(CnosDBLikeOperation op) { - super.visit((BinaryOperation) op); - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBVisitor.java b/src/sqlancer/cnosdb/CnosDBVisitor.java deleted file mode 100644 index 7c1af7224..000000000 --- a/src/sqlancer/cnosdb/CnosDBVisitor.java +++ /dev/null @@ -1,102 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBBetweenOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBFunction; -import sqlancer.cnosdb.ast.CnosDBInOperation; -import sqlancer.cnosdb.ast.CnosDBLikeOperation; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixText; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.ast.CnosDBSimilarTo; - -public interface CnosDBVisitor { - - static String asString(CnosDBExpression expr) { - CnosDBToStringVisitor visitor = new CnosDBToStringVisitor(); - visitor.visit(expr); - return visitor.get(); - } - - void visit(CnosDBConstant constant); - - void visit(CnosDBPostfixOperation op); - - void visit(CnosDBColumnValue c); - - void visit(CnosDBPrefixOperation op); - - void visit(CnosDBSelect op); - - void visit(CnosDBOrderByTerm op); - - void visit(CnosDBFunction f); - - void visit(CnosDBCastOperation cast); - - void visit(CnosDBBetweenOperation op); - - void visit(CnosDBInOperation op); - - void visit(CnosDBPostfixText op); - - void visit(CnosDBAggregate op); - - void visit(CnosDBFromTable from); - - void visit(CnosDBSubquery subquery); - - void visit(CnosDBBinaryLogicalOperation op); - - void visit(CnosDBLikeOperation op); - - void visit(CnosDBSimilarTo op); - - default void visit(CnosDBExpression expression) { - if (expression instanceof CnosDBConstant) { - visit((CnosDBConstant) expression); - } else if (expression instanceof CnosDBPostfixOperation) { - visit((CnosDBPostfixOperation) expression); - } else if (expression instanceof CnosDBColumnValue) { - visit((CnosDBColumnValue) expression); - } else if (expression instanceof CnosDBPrefixOperation) { - visit((CnosDBPrefixOperation) expression); - } else if (expression instanceof CnosDBSelect) { - visit((CnosDBSelect) expression); - } else if (expression instanceof CnosDBOrderByTerm) { - visit((CnosDBOrderByTerm) expression); - } else if (expression instanceof CnosDBFunction) { - visit((CnosDBFunction) expression); - } else if (expression instanceof CnosDBCastOperation) { - visit((CnosDBCastOperation) expression); - } else if (expression instanceof CnosDBBetweenOperation) { - visit((CnosDBBetweenOperation) expression); - } else if (expression instanceof CnosDBInOperation) { - visit((CnosDBInOperation) expression); - } else if (expression instanceof CnosDBAggregate) { - visit((CnosDBAggregate) expression); - } else if (expression instanceof CnosDBPostfixText) { - visit((CnosDBPostfixText) expression); - } else if (expression instanceof CnosDBSimilarTo) { - visit((CnosDBSimilarTo) expression); - } else if (expression instanceof CnosDBFromTable) { - visit((CnosDBFromTable) expression); - } else if (expression instanceof CnosDBSubquery) { - visit((CnosDBSubquery) expression); - } else if (expression instanceof CnosDBLikeOperation) { - visit((CnosDBLikeOperation) expression); - } else { - throw new AssertionError(expression); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAggregate.java b/src/sqlancer/cnosdb/ast/CnosDBAggregate.java deleted file mode 100644 index df30717b4..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBAggregate.java +++ /dev/null @@ -1,113 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBBugs; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; -import sqlancer.common.ast.FunctionNode; - -public class CnosDBAggregate extends FunctionNode - implements CnosDBExpression { - - public CnosDBAggregate(List args, CnosDBAggregateFunction func) { - super(func, args); - } - - public enum CnosDBAggregateFunction { - AVG(CnosDBDataType.DOUBLE), - MAX(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, - CnosDBDataType.UINT), - MIN(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, - CnosDBDataType.UINT), - COUNT(CnosDBDataType.INT) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.getRandomType() }; - } - }, - SUM(CnosDBDataType.INT, CnosDBDataType.DOUBLE, CnosDBDataType.UINT), APPROX_MEDIAN(CnosDBDataType.DOUBLE), - - VAR(CnosDBDataType.DOUBLE), VAR_SAMP(CnosDBDataType.DOUBLE), VAR_POP(CnosDBDataType.DOUBLE), - STDDEV(CnosDBDataType.DOUBLE), STDDEV_SAMP(CnosDBDataType.DOUBLE), STDDEV_POP(CnosDBDataType.DOUBLE), - COVAR(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - COVAR_SAMP(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.INT }; - } - }, - CORR(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - COVAR_POP(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - - APPROX_PERCENTILE_CONT(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - APPROX_PERCENTILE_CONT_WITH_WEIGHT(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - APPROX_DISTINCT(CnosDBDataType.UINT), GROUPING(CnosDBDataType.INT), ARRAY_AGG(CnosDBDataType.STRING); - - private final CnosDBDataType[] supportedReturnTypes; - - CnosDBAggregateFunction(CnosDBDataType... supportedReturnTypes) { - this.supportedReturnTypes = supportedReturnTypes.clone(); - } - - public static List getAggregates(CnosDBDataType type) { - List res = Stream.of(values()).filter(p -> p.supportsReturnType(type)) - .collect(Collectors.toList()); - if (CnosDBBugs.BUG786) { - res.removeAll(List.of(VAR, VAR_POP, VAR_SAMP, STDDEV, STDDEV_POP, STDDEV_SAMP, CORR, COVAR, COVAR_POP, - COVAR_SAMP, APPROX_PERCENTILE_CONT_WITH_WEIGHT, APPROX_DISTINCT, APPROX_PERCENTILE_CONT, - APPROX_PERCENTILE_CONT_WITH_WEIGHT, GROUPING, ARRAY_AGG)); - } - - return res; - } - - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { returnType }; - } - - public boolean supportsReturnType(CnosDBDataType returnType) { - return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType) - || supportedReturnTypes.length == 0; - } - - public CnosDBDataType getRandomReturnType() { - if (supportedReturnTypes.length == 0) { - return Randomly.fromOptions(CnosDBDataType.getRandomType()); - } else { - return Randomly.fromOptions(supportedReturnTypes); - } - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAlias.java b/src/sqlancer/cnosdb/ast/CnosDBAlias.java deleted file mode 100644 index 86bba199f..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBAlias.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.common.visitor.UnaryOperation; - -public class CnosDBAlias implements UnaryOperation, CnosDBExpression { - - private final CnosDBExpression expr; - private final String alias; - - public CnosDBAlias(CnosDBExpression expr, String alias) { - this.expr = expr; - this.alias = alias; - } - - @Override - public CnosDBExpression getExpression() { - return expr; - } - - @Override - public String getOperatorRepresentation() { - return " as " + alias; - } - - @Override - public OperatorKind getOperatorKind() { - return OperatorKind.POSTFIX; - } - - @Override - public boolean omitBracketsWhenPrinting() { - return true; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java deleted file mode 100644 index d0addced1..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java +++ /dev/null @@ -1,34 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public final class CnosDBBetweenOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final CnosDBExpression left; - private final CnosDBExpression right; - - public CnosDBBetweenOperation(CnosDBExpression expr, CnosDBExpression left, CnosDBExpression right) { - this.expr = expr; - this.left = left; - this.right = right; - } - - public CnosDBExpression getExpr() { - return expr; - } - - public CnosDBExpression getLeft() { - return left; - } - - public CnosDBExpression getRight() { - return right; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java deleted file mode 100644 index acf3e93d5..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java +++ /dev/null @@ -1,69 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; -import sqlancer.common.ast.BinaryOperatorNode; - -public class CnosDBBinaryArithmeticOperation extends BinaryOperatorNode - implements CnosDBExpression { - - public CnosDBBinaryArithmeticOperation(CnosDBExpression left, CnosDBExpression right, CnosDBBinaryOperator op) { - super(left, right, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.INT; - } - - public enum CnosDBBinaryOperator implements BinaryOperatorNode.Operator { - - ADDITION("+") { - }, - SUBTRACTION("-") { - }, - MULTIPLICATION("*") { - }, - DIVISION("/") { - - }, - MODULO("%") { - }, - EXPONENTIATION("^") { - }; - - private final String textRepresentation; - - CnosDBBinaryOperator(String textRepresentation) { - this.textRepresentation = textRepresentation; - } - - public static CnosDBBinaryOperator getRandom(CnosDBDataType dataType) { - List ops = new ArrayList<>(Arrays.asList(values())); - switch (dataType) { - case DOUBLE: - case UINT: - case STRING: - ops.remove(EXPONENTIATION); - ops.remove(MODULO); - break; - default: - break; - } - - return Randomly.fromList(ops); - } - - @Override - public String getTextRepresentation() { - return textRepresentation; - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java deleted file mode 100644 index af38849c9..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java +++ /dev/null @@ -1,57 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator; -import sqlancer.common.ast.BinaryOperatorNode; - -public class CnosDBBinaryComparisonOperation - extends BinaryOperatorNode implements CnosDBExpression { - - public CnosDBBinaryComparisonOperation(CnosDBExpression left, CnosDBExpression right, - CnosDBBinaryComparisonOperator op) { - super(left, right, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - public enum CnosDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { - EQUALS("=") { - }, - IS_DISTINCT("IS DISTINCT FROM") { - }, - IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { - }, - NOT_EQUALS("!=") { - }, - LESS("<") { - }, - LESS_EQUALS("<=") { - }, - GREATER(">") { - }, - GREATER_EQUALS(">=") { - - }; - - private final String textRepresentation; - - CnosDBBinaryComparisonOperator(String textRepresentation) { - this.textRepresentation = textRepresentation; - } - - public static CnosDBBinaryComparisonOperator getRandom() { - return Randomly.fromOptions(CnosDBBinaryComparisonOperator.values()); - } - - @Override - public String getTextRepresentation() { - return textRepresentation; - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java deleted file mode 100644 index bad8a3b75..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java +++ /dev/null @@ -1,33 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; -import sqlancer.common.ast.BinaryOperatorNode; - -public class CnosDBBinaryLogicalOperation extends BinaryOperatorNode - implements CnosDBExpression { - - public CnosDBBinaryLogicalOperation(CnosDBExpression left, CnosDBExpression right, BinaryLogicalOperator op) { - super(left, right, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - public enum BinaryLogicalOperator implements BinaryOperatorNode.Operator { - AND, OR; - - public static BinaryLogicalOperator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return toString(); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java b/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java deleted file mode 100644 index 41db62d81..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java +++ /dev/null @@ -1,60 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.cnosdb.CnosDBCompoundDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBCastOperation implements CnosDBExpression { - - private final CnosDBExpression expression; - private final CnosDBCompoundDataType type; - - public CnosDBCastOperation(CnosDBExpression expression, CnosDBCompoundDataType type) { - if (expression == null) { - throw new AssertionError(); - } - this.expression = expression; - this.type = type; - } - - public static List canCastTo(CnosDBDataType dataType) { - List options = new ArrayList<>(Arrays.asList(CnosDBDataType.values())); - - switch (dataType) { - case UINT: - case BOOLEAN: - case DOUBLE: - options.remove(CnosDBDataType.TIMESTAMP); - break; - case TIMESTAMP: - options.remove(CnosDBDataType.BOOLEAN); - options.remove(CnosDBDataType.UINT); - options.remove(CnosDBDataType.DOUBLE); - break; - default: - break; - } - return options; - } - - @Override - public CnosDBDataType getExpressionType() { - return type.getDataType(); - } - - public CnosDBExpression getExpression() { - return expression; - } - - public CnosDBDataType getType() { - return type.getDataType(); - } - - public CnosDBCompoundDataType getCompoundType() { - return type; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java b/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java deleted file mode 100644 index f90b6120f..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java +++ /dev/null @@ -1,27 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBColumnValue implements CnosDBExpression { - - private final CnosDBColumn c; - - public CnosDBColumnValue(CnosDBColumn c) { - this.c = c; - } - - public static CnosDBColumnValue create(CnosDBColumn c) { - return new CnosDBColumnValue(c); - } - - @Override - public CnosDBDataType getExpressionType() { - return c.getType(); - } - - public CnosDBColumn getColumn() { - return c; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java b/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java deleted file mode 100644 index 6821f83b8..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryNode; - -public class CnosDBConcatOperation extends BinaryNode implements CnosDBExpression { - - public CnosDBConcatOperation(CnosDBExpression left, CnosDBExpression right) { - super(left, right); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.STRING; - } - - @Override - public String getOperatorRepresentation() { - return "||"; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConstant.java b/src/sqlancer/cnosdb/ast/CnosDBConstant.java deleted file mode 100644 index 42ecd3908..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBConstant.java +++ /dev/null @@ -1,520 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.math.BigDecimal; -import java.text.SimpleDateFormat; -import java.util.Date; - -import sqlancer.IgnoreMeException; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public abstract class CnosDBConstant implements CnosDBExpression { - - public static CnosDBConstant createNullConstant() { - return new CnosDBNullConstant(); - } - - public static CnosDBConstant createIntConstant(long val) { - return new IntConstant(val, false); - } - - public static CnosDBConstant createBooleanConstant(boolean val) { - return new BooleanConstant(val); - } - - public static CnosDBConstant createFalse() { - return createBooleanConstant(false); - } - - public static CnosDBConstant createTrue() { - return createBooleanConstant(true); - } - - public static CnosDBConstant createStringConstant(String string) { - return new StringConstant(string); - } - - public static CnosDBConstant createDoubleConstant(double val) { - return new DoubleConstant(val); - } - - public static CnosDBConstant createUintConstant(long val) { - return new IntConstant(val, true); - } - - public static CnosDBConstant createTimeStampConstant(long val) { - return new TimeStampConstant(val); - } - - public abstract String getTextRepresentation(); - - public String asString() { - throw new UnsupportedOperationException(this.toString()); - } - - public boolean isString() { - return false; - } - - public boolean isNull() { - return false; - } - - public boolean asBoolean() { - throw new UnsupportedOperationException(this.toString()); - } - - public long asInt() { - throw new UnsupportedOperationException(this.toString()); - } - - public double asDouble() { - throw new UnsupportedOperationException(this.toString()); - } - - public boolean isBoolean() { - return false; - } - - public abstract CnosDBConstant isEquals(CnosDBConstant rightVal); - - public boolean isInt() { - return false; - } - - protected abstract CnosDBConstant isLessThan(CnosDBConstant rightVal); - - @Override - public String toString() { - return getTextRepresentation(); - } - - public abstract CnosDBConstant cast(CnosDBDataType type); - - public static class BooleanConstant extends CnosDBConstant { - - private final boolean value; - - public BooleanConstant(boolean value) { - this.value = value; - } - - @Override - public String getTextRepresentation() { - return value ? "TRUE" : "FALSE"; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - @Override - public boolean asBoolean() { - return value; - } - - @Override - public boolean isBoolean() { - return true; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return CnosDBConstant.createBooleanConstant(value == rightVal.asBoolean()); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(value == rightVal.cast(CnosDBDataType.BOOLEAN).asBoolean()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isString()) { - return isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); - } else { - assert rightVal.isBoolean(); - return CnosDBConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - switch (type) { - case BOOLEAN: - return this; - case INT: - return CnosDBConstant.createIntConstant(value ? 1 : 0); - case UINT: - return CnosDBConstant.createUintConstant(value ? 1 : 0); - case STRING: - return CnosDBConstant.createStringConstant(value ? "true" : "false"); - default: - return null; - } - } - - } - - public static class CnosDBNullConstant extends CnosDBConstant { - - @Override - public String getTextRepresentation() { - return "NULL"; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - - @Override - public boolean isNull() { - return true; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - return CnosDBConstant.createNullConstant(); - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - return CnosDBConstant.createNullConstant(); - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - return CnosDBConstant.createNullConstant(); - } - } - - public static class StringConstant extends CnosDBConstant { - - private final String value; - - public StringConstant(String value) { - this.value = value; - } - - @Override - public String getTextRepresentation() { - return String.format("'%s'", value.replace("'", "''")); - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isInt()) { - return cast(CnosDBDataType.INT).isEquals(rightVal.cast(CnosDBDataType.INT)); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal.cast(CnosDBDataType.BOOLEAN)); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isInt()) { - return cast(CnosDBDataType.INT).isLessThan(rightVal.cast(CnosDBDataType.INT)); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - if (type == CnosDBDataType.STRING) { - return this; - } - String s = value.trim(); - switch (type) { - case BOOLEAN: - try { - return CnosDBConstant.createBooleanConstant(Long.parseLong(s) != 0); - } catch (NumberFormatException ignored) { - } - switch (s.toUpperCase()) { - case "T": - case "TR": - case "TRU": - case "TRUE": - case "1": - case "YES": - case "YE": - case "Y": - case "ON": - return CnosDBConstant.createTrue(); - case "F": - case "FA": - case "FAL": - case "FALS": - case "FALSE": - case "N": - case "NO": - case "OF": - case "OFF": - default: - return CnosDBConstant.createFalse(); - } - case INT: - try { - return CnosDBConstant.createIntConstant(Long.parseLong(s)); - } catch (NumberFormatException e) { - return CnosDBConstant.createIntConstant(-1); - } - case UINT: - try { - return CnosDBConstant.createUintConstant(Long.parseUnsignedLong(s)); - } catch (NumberFormatException e) { - return CnosDBConstant.createUintConstant(0); - } - case DOUBLE: - try { - return CnosDBConstant.createDoubleConstant(Double.parseDouble(s)); - } catch (NumberFormatException e) { - return CnosDBConstant.createDoubleConstant(0.0); - } - - default: - return null; - } - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.STRING; - } - - @Override - public boolean isString() { - return true; - } - - @Override - public String asString() { - return value; - } - - } - - public static class IntConstant extends CnosDBConstant { - - private final long val; - private final boolean unsigned; - - public IntConstant(long val, boolean unsigned) { - this.val = val; - this.unsigned = unsigned; - } - - @Override - public String getTextRepresentation() { - if (unsigned) { - return Long.toUnsignedString(val); - } else { - return String.valueOf(val); - } - } - - @Override - public CnosDBDataType getExpressionType() { - if (unsigned) { - return CnosDBDataType.UINT; - } - return CnosDBDataType.INT; - } - - @Override - public long asInt() { - return val; - } - - @Override - public double asDouble() { - return val; - } - - @Override - public boolean isInt() { - return true; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); - } else if (rightVal.isInt()) { - return CnosDBConstant.createBooleanConstant(val == rightVal.asInt()); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.INT).asInt()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isInt()) { - return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); - } else if (rightVal.isBoolean()) { - throw new AssertionError(rightVal); - } else if (rightVal.getExpressionType() == CnosDBDataType.UINT) { - return CnosDBConstant.createBooleanConstant(Long.compareUnsigned(val, rightVal.asInt()) < 0); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.INT).asInt()); - } else { - throw new IgnoreMeException(); - } - - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - switch (type) { - case BOOLEAN: - return CnosDBConstant.createBooleanConstant(val != 0); - case INT: - return CnosDBConstant.createIntConstant(val); - case STRING: - return CnosDBConstant.createStringConstant(String.valueOf(val)); - case UINT: - return CnosDBConstant.createUintConstant(val); - case DOUBLE: - return CnosDBConstant.createDoubleConstant(val); - default: - return null; - } - } - } - - public static class TimeStampConstant extends CnosDBConstant { - final long val; - - TimeStampConstant(long time) { - val = time; - } - - @Override - public String getTextRepresentation() { - return "CAST (" + val + " AS TIMESTAMP)"; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return createNullConstant(); - } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { - return createBooleanConstant(val == rightVal.asInt()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { - return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - switch (type) { - case INT: - return createIntConstant(val); - case STRING: - final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); - return CnosDBConstant.createStringConstant(dateFormat.format(new Date(val))); - default: - return null; - } - } - - @Override - public long asInt() { - return val; - } - - } - - public static class DoubleConstant extends CnosDBConstant { - - private final double val; - - public DoubleConstant(double val) { - this.val = val; - } - - @Override - public String getTextRepresentation() { - if (Double.isFinite(val)) { - BigDecimal bigDecimal = new BigDecimal(val); - return bigDecimal.toPlainString(); - } else { - return String.valueOf(0.0); - } - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.DOUBLE; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal); - } else { - return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); - } - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); - } else { - return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - return null; - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBExpression.java b/src/sqlancer/cnosdb/ast/CnosDBExpression.java deleted file mode 100644 index 63997a0f5..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBExpression.java +++ /dev/null @@ -1,14 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public interface CnosDBExpression { - - default CnosDBDataType getExpressionType() { - return null; - } - - default CnosDBConstant getExpectedValue() { - throw new AssertionError("Not impl"); - } -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunction.java b/src/sqlancer/cnosdb/ast/CnosDBFunction.java deleted file mode 100644 index 7a35d703e..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBFunction.java +++ /dev/null @@ -1,30 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBFunction implements CnosDBExpression { - - private final String func; - private final CnosDBExpression[] args; - private final CnosDBDataType returnType; - - public CnosDBFunction(CnosDBFunctionWithUnknownResult f, CnosDBDataType returnType, CnosDBExpression... args) { - this.func = f.getName(); - this.returnType = returnType; - this.args = args.clone(); - } - - public String getFunctionName() { - return func; - } - - public CnosDBExpression[] getArguments() { - return args.clone(); - } - - @Override - public CnosDBDataType getExpressionType() { - return returnType; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java b/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java deleted file mode 100644 index 485f2309d..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java +++ /dev/null @@ -1,104 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import sqlancer.cnosdb.CnosDBBugs; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; - -public enum CnosDBFunctionWithUnknownResult { - - // String functions - ASCII("ascii", CnosDBDataType.INT, CnosDBDataType.STRING), - BTRIM("btrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - CHAR_LENGTH("char_length", CnosDBDataType.INT, CnosDBDataType.STRING), - CHARACTER_LENGTH("character_length", CnosDBDataType.INT, CnosDBDataType.STRING), - CONCAT("concat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - CONCAT_WS("concat_ws", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - CHR("chr", CnosDBDataType.STRING, CnosDBDataType.INT), - BIT_LENGTH("bit_length", CnosDBDataType.INT, CnosDBDataType.STRING), - INITCAP("initcap", CnosDBDataType.STRING, CnosDBDataType.STRING), - - LEFT("left", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - LENGTH("length", CnosDBDataType.UINT, CnosDBDataType.STRING), - LOWER("lower", CnosDBDataType.STRING, CnosDBDataType.STRING), - UPPER("upper", CnosDBDataType.STRING, CnosDBDataType.STRING), - LPAD3("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), - LPAD2("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - RPAD3("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), - RPAD2("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - LTRIM("ltrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - OCTET_LENGTH("octet_length", CnosDBDataType.INT, CnosDBDataType.STRING), - // REPEAT("repeat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - REPLACE("replace", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - REVERSE("reverse", CnosDBDataType.STRING, CnosDBDataType.STRING), - RIGHT("right", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - RTRIM("rtrim", CnosDBDataType.STRING, CnosDBDataType.STRING), - SPLIT_PART("split_part", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - STARTS_WITH("starts_with", CnosDBDataType.BOOLEAN, CnosDBDataType.STRING, CnosDBDataType.STRING), - STRPOS("strpos", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.STRING), - SUBSTR("substr", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.INT), - TRANSLATE("translate", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - MD5("md5", CnosDBDataType.STRING, CnosDBDataType.STRING), - // mathematical functions - ABS("abs", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - CEIL("ceil", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - EXP("exp", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), LN("ln", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - LOG2("log2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - LOG10("log10", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - POWER("power", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ROUND("round", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - TRUNC("trunc", CnosDBDataType.DOUBLE, CnosDBDataType.INT), - FLOOR("floor", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - SIGNUM("signum", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ACOS("acos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ASIN("asin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ATAN2("atan2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - COS("cos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), SIN("sin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - SQRT("sqrt", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - TAN("tan", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - DATE_PART("date_part", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP), - TO_TIMESTAMP("to_timestamp", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), - TO_TIMESTAMP_MILLIS("to_timestamp_millis", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), - TO_TIMESTAMP_MICROS("to_timestamp_micros", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), - TO_TIMESTAMP_SECONDS("to_timestamp_seconds", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT); - - private final String functionName; - private final CnosDBDataType returnType; - private final CnosDBDataType[] argTypes; - - CnosDBFunctionWithUnknownResult(String functionName, CnosDBDataType returnType, CnosDBDataType... indexType) { - this.functionName = functionName; - this.returnType = returnType; - this.argTypes = indexType.clone(); - - } - - public static List getSupportedFunctions(CnosDBDataType type) { - List res = Stream.of(values()) - .filter(function -> function.isCompatibleWithReturnType(type)).collect(Collectors.toList()); - if (CnosDBBugs.BUG3547) { - res.removeAll(List.of(TO_TIMESTAMP, TO_TIMESTAMP_MICROS, TO_TIMESTAMP_MILLIS, TO_TIMESTAMP_SECONDS)); - } - return res; - } - - public boolean isCompatibleWithReturnType(CnosDBDataType t) { - return t == returnType; - } - - public CnosDBExpression[] getArguments(CnosDBDataType ignore, CnosDBExpressionGenerator gen, int depth) { - CnosDBExpression[] args = new CnosDBExpression[argTypes.length]; - for (int i = 0; i < args.length; i++) { - args[i] = gen.generateExpression(depth, argTypes[i]); - } - return args; - } - - public String getName() { - return functionName; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBInOperation.java b/src/sqlancer/cnosdb/ast/CnosDBInOperation.java deleted file mode 100644 index c0ffd34ed..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBInOperation.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.List; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBInOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final List listElements; - private final boolean isTrue; - - public CnosDBInOperation(CnosDBExpression expr, List listElements, boolean isTrue) { - this.expr = expr; - this.listElements = listElements; - this.isTrue = isTrue; - } - - public CnosDBExpression getExpr() { - return expr; - } - - public List getListElements() { - return listElements; - } - - public boolean isTrue() { - return isTrue; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBJoin.java b/src/sqlancer/cnosdb/ast/CnosDBJoin.java deleted file mode 100644 index eea88466f..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBJoin.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBJoin implements CnosDBExpression { - - private final CnosDBExpression tableReference; - private final CnosDBExpression onClause; - private final CnosDBJoinType type; - - public CnosDBJoin(CnosDBExpression tableReference, CnosDBExpression onClause, CnosDBJoinType type) { - this.tableReference = tableReference; - this.onClause = onClause; - this.type = type; - } - - public CnosDBExpression getTableReference() { - return tableReference; - } - - public CnosDBExpression getOnClause() { - return onClause; - } - - public CnosDBJoinType getType() { - return type; - } - - @Override - public CnosDBDataType getExpressionType() { - throw new AssertionError(); - } - - public enum CnosDBJoinType { - INNER, LEFT, RIGHT, FULL; - // now not support - // CROSS; - - public static CnosDBJoinType getRandom() { - return Randomly.fromOptions(values()); - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java b/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java deleted file mode 100644 index 616cd39ee..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryNode; - -public class CnosDBLikeOperation extends BinaryNode implements CnosDBExpression { - - public CnosDBLikeOperation(CnosDBExpression left, CnosDBExpression right) { - super(left, right); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - @Override - public String getOperatorRepresentation() { - return "LIKE"; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java b/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java deleted file mode 100644 index de5812d76..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java +++ /dev/null @@ -1,37 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBOrderByTerm implements CnosDBExpression { - - private final CnosDBOrder order; - private final CnosDBExpression expr; - - public CnosDBOrderByTerm(CnosDBExpression expr, CnosDBOrder order) { - this.expr = expr; - this.order = order; - } - - public CnosDBOrder getOrder() { - return order; - } - - public CnosDBExpression getExpr() { - return expr; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - - public enum CnosDBOrder { - ASC, DESC; - - public static CnosDBOrder getRandomOrder() { - return Randomly.fromOptions(CnosDBOrder.values()); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java deleted file mode 100644 index f37621f44..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java +++ /dev/null @@ -1,97 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryOperatorNode.Operator; - -public class CnosDBPostfixOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final String operatorTextRepresentation; - - public CnosDBPostfixOperation(CnosDBExpression expr, PostfixOperator op) { - this.expr = expr; - this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); - } - - public static CnosDBExpression create(CnosDBExpression expr, PostfixOperator op) { - return new CnosDBPostfixOperation(expr, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - public String getOperatorTextRepresentation() { - return operatorTextRepresentation; - } - - public CnosDBExpression getExpression() { - return expr; - } - - public enum PostfixOperator implements Operator { - IS_NULL("IS NULL"/* , "ISNULL" */) { - @Override - public CnosDBDataType[] getInputDataTypes() { - return CnosDBDataType.values(); - } - - }, - IS_UNKNOWN("IS UNKNOWN") { - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - }, - - IS_NOT_NULL("IS NOT NULL"/* "NOTNULL" */) { - - @Override - public CnosDBDataType[] getInputDataTypes() { - return CnosDBDataType.values(); - } - - }, - IS_NOT_UNKNOWN("IS NOT UNKNOWN") { - - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - }, - IS_TRUE("IS TRUE") { - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - - }, - IS_FALSE("IS FALSE") { - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - - }; - - private final String[] textRepresentations; - - PostfixOperator(String... textRepresentations) { - this.textRepresentations = textRepresentations.clone(); - } - - public static PostfixOperator getRandom() { - return Randomly.fromOptions(values()); - } - - public abstract CnosDBDataType[] getInputDataTypes(); - - @Override - public String getTextRepresentation() { - return toString(); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java deleted file mode 100644 index 241fab89a..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java +++ /dev/null @@ -1,29 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBPostfixText implements CnosDBExpression { - - private final CnosDBExpression expr; - private final String text; - private final CnosDBDataType type; - - public CnosDBPostfixText(CnosDBExpression expr, String text, CnosDBDataType type) { - this.expr = expr; - this.text = text; - this.type = type; - } - - public CnosDBExpression getExpr() { - return expr; - } - - public String getText() { - return text; - } - - @Override - public CnosDBDataType getExpressionType() { - return type; - } -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java deleted file mode 100644 index db37f0089..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java +++ /dev/null @@ -1,73 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryOperatorNode.Operator; - -public class CnosDBPrefixOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final PrefixOperator op; - - public CnosDBPrefixOperation(CnosDBExpression expr, PrefixOperator op) { - this.expr = expr; - this.op = op; - } - - @Override - public CnosDBDataType getExpressionType() { - return op.getExpressionType(); - } - - public CnosDBDataType[] getInputDataTypes() { - return op.dataTypes; - } - - public String getTextRepresentation() { - return op.textRepresentation; - } - - public CnosDBExpression getExpression() { - return expr; - } - - public enum PrefixOperator implements Operator { - NOT("NOT", CnosDBDataType.BOOLEAN) { - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - }, - UNARY_PLUS("+", CnosDBDataType.INT) { - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.INT; - } - - }, - UNARY_MINUS("-", CnosDBDataType.INT) { - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.INT; - } - - }; - - private final String textRepresentation; - private final CnosDBDataType[] dataTypes; - - PrefixOperator(String textRepresentation, CnosDBDataType... dataTypes) { - this.textRepresentation = textRepresentation; - this.dataTypes = dataTypes.clone(); - } - - public abstract CnosDBDataType getExpressionType(); - - @Override - public String getTextRepresentation() { - return toString(); - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSelect.java b/src/sqlancer/cnosdb/ast/CnosDBSelect.java deleted file mode 100644 index 0db657f19..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBSelect.java +++ /dev/null @@ -1,102 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.Collections; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.common.ast.SelectBase; - -public class CnosDBSelect extends SelectBase implements CnosDBExpression { - - private SelectType selectOption = SelectType.ALL; - private List joinClauses = Collections.emptyList(); - private CnosDBExpression distinctOnClause; - - public void setSelectType(SelectType fromOptions) { - this.setSelectOption(fromOptions); - } - - public SelectType getSelectOption() { - return selectOption; - } - - public void setSelectOption(SelectType fromOptions) { - this.selectOption = fromOptions; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - - public List getJoinClauses() { - return joinClauses; - } - - public void setJoinClauses(List joinStatements) { - this.joinClauses = joinStatements; - - } - - public CnosDBExpression getDistinctOnClause() { - return distinctOnClause; - } - - public void setDistinctOnClause(CnosDBExpression distinctOnClause) { - if (selectOption != SelectType.DISTINCT) { - throw new IllegalArgumentException(); - } - this.distinctOnClause = distinctOnClause; - } - - public enum SelectType { - DISTINCT, ALL; - - public static SelectType getRandom() { - return Randomly.fromOptions(values()); - } - } - - public static class CnosDBFromTable implements CnosDBExpression { - private final CnosDBTable t; - - public CnosDBFromTable(CnosDBTable t) { - this.t = t; - } - - public CnosDBTable getTable() { - return t; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - } - - public static class CnosDBSubquery implements CnosDBExpression { - private final CnosDBSelect s; - private final String name; - - public CnosDBSubquery(CnosDBSelect s, String name) { - this.s = s; - this.name = name; - } - - public CnosDBSelect getSelect() { - return s; - } - - public String getName() { - return name; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java b/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java deleted file mode 100644 index 9e3467ada..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java +++ /dev/null @@ -1,28 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBSimilarTo implements CnosDBExpression { - - private final CnosDBExpression string; - private final CnosDBExpression similarTo; - - public CnosDBSimilarTo(CnosDBExpression string, CnosDBExpression similarTo) { - this.string = string; - this.similarTo = similarTo; - } - - public CnosDBExpression getString() { - return string; - } - - public CnosDBExpression getSimilarTo() { - return similarTo; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - -} diff --git a/src/sqlancer/cnosdb/client/CnosDBClient.java b/src/sqlancer/cnosdb/client/CnosDBClient.java deleted file mode 100644 index ccc9dcc16..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBClient.java +++ /dev/null @@ -1,110 +0,0 @@ -package sqlancer.cnosdb.client; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; - -import org.apache.commons.codec.binary.Base64; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; - -import com.arangodb.internal.util.IOUtils; - -public class CnosDBClient { - private final String userName; - private final String password; - private final String host; - private final int port; - - private final String database; - private final CloseableHttpClient client; - - public CnosDBClient(String host, int port, String userName, String password, String database) { - this.host = host; - this.port = port; - this.userName = userName; - this.password = password; - this.database = database; - this.client = HttpClientBuilder.create().build(); - } - - private String url() { - return "http://" + host + ":" + port + "/api/v1/"; - } - - public String ping() throws Exception { - HttpGet httpGet = new HttpGet(this.url() + "ping"); - httpGet.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); - CloseableHttpResponse resp = client.execute(httpGet); - - String content = IOUtils.toString(resp.getEntity().getContent()); - resp.close(); - return content; - } - - public CnosDBResultSet executeQuery(String query) throws Exception { - HttpUriRequest request = createRequest(query); - CloseableHttpResponse resp = client.execute(request); - String text = IOUtils.toString(resp.getEntity().getContent()); - if (resp.getStatusLine().getStatusCode() != 200) { - resp.close(); - throw new CnosDBException(database + ":" + query + ";\n" + text); - } - resp.close(); - InputStream stream = new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); - - return new CnosDBResultSet(new InputStreamReader(stream)); - } - - public boolean execute(String query) throws Exception { - HttpUriRequest request = createRequest(query); - CloseableHttpResponse resp = client.execute(request); - if (resp.getStatusLine().getStatusCode() != 200) { - String res = IOUtils.toString(resp.getEntity().getContent()); - resp.close(); - throw new CnosDBException(query + res); - } - resp.close(); - return true; - } - - public void close() throws IOException { - client.close(); - } - - public String getDatabase() { - return this.database; - } - - private String getAuth() { - String auth = userName + ":" + password; - byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.ISO_8859_1)); - return "Basic " + new String(encodedAuth); - - } - - private HttpUriRequest createRequest(String query) throws URISyntaxException, UnsupportedEncodingException { - - URIBuilder builder = new URIBuilder(this.url() + "sql"); - builder.setParameter("db", database); - builder.setParameter("pretty", "true"); - HttpPost httpPost = new HttpPost(builder.build()); - - httpPost.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); - StringEntity stringEntity = new StringEntity(query); - httpPost.setEntity(stringEntity); - return httpPost; - } - -} diff --git a/src/sqlancer/cnosdb/client/CnosDBConnection.java b/src/sqlancer/cnosdb/client/CnosDBConnection.java deleted file mode 100644 index 9277f203b..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBConnection.java +++ /dev/null @@ -1,27 +0,0 @@ -package sqlancer.cnosdb.client; - -import java.io.IOException; - -import sqlancer.SQLancerDBConnection; - -public class CnosDBConnection implements SQLancerDBConnection { - private final CnosDBClient client; - - public CnosDBConnection(CnosDBClient client) { - this.client = client; - } - - @Override - public String getDatabaseVersion() throws Exception { - return client.ping(); - } - - public CnosDBClient getClient() { - return client; - } - - @Override - public void close() throws IOException { - client.close(); - } -} diff --git a/src/sqlancer/cnosdb/client/CnosDBException.java b/src/sqlancer/cnosdb/client/CnosDBException.java deleted file mode 100644 index a1055e90b..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBException.java +++ /dev/null @@ -1,9 +0,0 @@ -package sqlancer.cnosdb.client; - -public class CnosDBException extends RuntimeException { - private static final long serialVersionUID = 1L; - - CnosDBException(String message) { - super(message); - } -} diff --git a/src/sqlancer/cnosdb/client/CnosDBResultSet.java b/src/sqlancer/cnosdb/client/CnosDBResultSet.java deleted file mode 100644 index 877b6ba5d..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBResultSet.java +++ /dev/null @@ -1,52 +0,0 @@ -package sqlancer.cnosdb.client; - -import java.io.Reader; -import java.sql.SQLException; -import java.util.Iterator; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; - -import sqlancer.IgnoreMeException; - -public class CnosDBResultSet { - private final Iterator records; - private CSVRecord next; - - public CnosDBResultSet(Reader in) throws Exception { - Iterable records = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).build() - .parse(in); - this.records = records.iterator(); - } - - public void close() { - } - - public boolean next() throws SQLException { - if (records.hasNext()) { - next = records.next(); - return true; - } - return false; - } - - public int getInt(int i) throws SQLException { - return Integer.parseInt(next.get(i - 1)); - } - - public String getString(int i) throws SQLException { - return next.get(i - 1); - } - - public long getLong(int i) throws SQLException { - if (next.get(i - 1).isEmpty()) { - throw new IgnoreMeException(); - } - return Long.parseLong(next.get(i - 1)); - } - - // public boolean getBool(int i) throws Exception { - // return Boolean.parseBoolean(getString(i)); - // } - -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBCommon.java b/src/sqlancer/cnosdb/gen/CnosDBCommon.java deleted file mode 100644 index 6c7b0bba7..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBCommon.java +++ /dev/null @@ -1,31 +0,0 @@ -package sqlancer.cnosdb.gen; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public final class CnosDBCommon { - - private CnosDBCommon() { - } - - public static void appendDataType(CnosDBDataType type, StringBuilder sb) throws AssertionError { - switch (type) { - case BOOLEAN: - sb.append("BOOLEAN"); - break; - case INT: - sb.append("BIGINT"); - break; - case STRING: - sb.append("STRING"); - break; - case DOUBLE: - sb.append("DOUBLE"); - break; - case UINT: - sb.append("BIGINT UNSIGNED"); - break; - default: - throw new AssertionError(type); - } - } -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java deleted file mode 100644 index 121f78254..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java +++ /dev/null @@ -1,461 +0,0 @@ -package sqlancer.cnosdb.gen; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBCompoundDataType; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; -import sqlancer.cnosdb.ast.CnosDBBetweenOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; -import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConcatOperation; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBFunction; -import sqlancer.cnosdb.ast.CnosDBFunctionWithUnknownResult; -import sqlancer.cnosdb.ast.CnosDBInOperation; -import sqlancer.cnosdb.ast.CnosDBLikeOperation; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm.CnosDBOrder; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; -import sqlancer.cnosdb.ast.CnosDBSimilarTo; -import sqlancer.common.gen.ExpressionGenerator; - -public class CnosDBExpressionGenerator implements ExpressionGenerator { - - private final int maxDepth; - - private final Randomly r; - - private List columns; - - private boolean allowAggregateFunctions; - - public CnosDBExpressionGenerator(CnosDBGlobalState globalState) { - this.r = globalState.getRandomly(); - this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); - } - - public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, CnosDBDataType type) { - return new CnosDBExpressionGenerator(globalState).generateExpression(0, type); - } - - private static CnosDBCompoundDataType getCompoundDataType(CnosDBDataType type) { - return CnosDBCompoundDataType.create(type); - } - - public static CnosDBExpression generateConstant(Randomly r, CnosDBDataType type) { - if (Randomly.getBooleanWithSmallProbability()) { - return CnosDBConstant.createNullConstant(); - } - switch (type) { - case INT: - return CnosDBConstant.createIntConstant(r.getInteger()); - case UINT: - return CnosDBConstant.createUintConstant(r.getPositiveInteger()); - case TIMESTAMP: - return CnosDBConstant.createTimeStampConstant(r.getPositiveIntegerNotNull()); - case BOOLEAN: - return CnosDBConstant.createBooleanConstant(Randomly.getBoolean()); - case STRING: - return CnosDBConstant.createStringConstant(r.getString()); - case DOUBLE: - return CnosDBConstant.createDoubleConstant(r.getDouble()); - default: - throw new AssertionError(type); - } - } - - public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns, - CnosDBDataType type) { - return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); - } - - public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns) { - return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0); - } - - public CnosDBExpressionGenerator setColumns(List columns) { - this.columns = columns; - return this; - } - - public CnosDBExpression generateExpression(int depth) { - return generateExpression(depth, CnosDBDataType.getRandomType()); - } - - public List generateOrderBy() { - List orderBys = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber(); i++) { - orderBys.add(new CnosDBOrderByTerm(CnosDBColumnValue.create(Randomly.fromList(columns)), - CnosDBOrder.getRandomOrder())); - } - return orderBys; - } - - private CnosDBExpression generateFunctionWithUnknownResult(int depth, CnosDBDataType type) { - List supportedFunctions = CnosDBFunctionWithUnknownResult - .getSupportedFunctions(type); - if (supportedFunctions.isEmpty()) { - throw new IgnoreMeException(); - } - CnosDBFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); - return new CnosDBFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); - } - - private CnosDBExpression generateBooleanExpression(int depth) { - List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); - BooleanExpression option = Randomly.fromList(validOptions); - switch (option) { - case POSTFIX_OPERATOR: - PostfixOperator random = PostfixOperator.getRandom(); - return CnosDBPostfixOperation - .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); - case IN_OPERATION: - return inOperation(depth + 1); - case NOT: - return new CnosDBPrefixOperation(generateExpression(depth + 1, CnosDBDataType.BOOLEAN), PrefixOperator.NOT); - case BINARY_LOGICAL_OPERATOR: - CnosDBExpression first = generateExpression(depth + 1, CnosDBDataType.BOOLEAN); - int nr = Randomly.smallNumber() + 1; - for (int i = 0; i < nr; i++) { - first = new CnosDBBinaryLogicalOperation(first, generateExpression(depth + 1, CnosDBDataType.BOOLEAN), - BinaryLogicalOperator.getRandom()); - } - return first; - case BINARY_COMPARISON: - CnosDBDataType dataType = getMeaningfulType(); - return generateComparison(depth, dataType); - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.BOOLEAN); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.BOOLEAN); - case LIKE: - return new CnosDBLikeOperation(generateExpression(depth + 1, CnosDBDataType.STRING), - generateExpression(depth + 1, CnosDBDataType.STRING)); - case BETWEEN: - CnosDBDataType type = getMeaningfulType(); - return new CnosDBBetweenOperation(generateExpression(depth + 1, type), generateExpression(depth + 1, type), - generateExpression(depth + 1, type)); - case SIMILAR_TO: - return new CnosDBSimilarTo(generateExpression(depth + 1, CnosDBDataType.STRING), - generateExpression(depth + 1, CnosDBDataType.STRING)); - default: - throw new AssertionError(); - } - } - - private CnosDBDataType getMeaningfulType() { - // make it more likely that the expression does not only consist of constant - // expressions - if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { - return CnosDBDataType.getRandomType(); - } else { - return Randomly.fromList(columns).getType(); - } - } - - private CnosDBExpression generateFunction(int depth, CnosDBDataType type) { - return generateFunctionWithUnknownResult(depth, type); - } - - private CnosDBExpression generateComparison(int depth, CnosDBDataType dataType) { - CnosDBExpression leftExpr = generateExpression(depth + 1, dataType); - CnosDBExpression rightExpr = generateExpression(depth + 1, dataType); - return getComparison(leftExpr, rightExpr); - } - - private CnosDBExpression getComparison(CnosDBExpression leftExpr, CnosDBExpression rightExpr) { - return new CnosDBBinaryComparisonOperation(leftExpr, rightExpr, - CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator.getRandom()); - } - - private CnosDBExpression inOperation(int depth) { - CnosDBDataType type = CnosDBDataType.getRandomType(); - CnosDBExpression leftExpr = generateExpression(depth + 1, type); - List rightExpr = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - rightExpr.add(generateConstant(new Randomly(), type)); - } - return new CnosDBInOperation(leftExpr, rightExpr, Randomly.getBoolean()); - } - - public CnosDBExpression generateExpression(int depth, CnosDBDataType originalType) { - return generateExpressionInternal(depth, originalType); - } - - private CnosDBExpression generateExpressionInternal(int depth, CnosDBDataType dataType) throws AssertionError { - if (allowAggregateFunctions && Randomly.getBoolean()) { - return getAggregate(dataType); - } - - if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { - // generic expression - if (Randomly.getBoolean() || depth > maxDepth) { - if (Randomly.getBooleanWithRatherLowProbability()) { - return generateConstant(r, dataType); - } else { - if (filterColumns(dataType).isEmpty()) { - return generateConstant(r, dataType); - } else { - return createColumnOfType(dataType); - } - } - } else { - if (Randomly.getBoolean()) { - return generateCastExpression(depth + 1, dataType); - } else { - return generateFunctionWithUnknownResult(depth, dataType); - } - } - } else { - switch (dataType) { - case BOOLEAN: - return generateBooleanExpression(depth); - case INT: - return generateIntExpression(depth); - case UINT: - return generateUIntExpression(depth); - case STRING: - return generateStringExpression(depth); - case DOUBLE: - return generateFloatExpression(depth); - case TIMESTAMP: - return generateTimeStampExpression(depth); - default: - throw new AssertionError(dataType); - } - } - } - - private CnosDBExpression generateStringExpression(int depth) { - StringExpression option; - List validOptions = new ArrayList<>(Arrays.asList(StringExpression.values())); - option = Randomly.fromList(validOptions); - - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.STRING); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.STRING); - case CONCAT: - return generateConcat(depth); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateConcat(int depth) { - CnosDBExpression left = generateExpression(depth + 1, CnosDBDataType.STRING); - CnosDBExpression right = generateExpression(depth + 1); - return new CnosDBConcatOperation(left, right); - } - - private CnosDBExpression generateIntExpression(int depth) { - IntExpression option; - option = Randomly.fromOptions(IntExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.INT); - case UNARY_OPERATION: - CnosDBExpression intExpression = generateExpression(depth + 1, CnosDBDataType.INT); - return new CnosDBPrefixOperation(intExpression, - Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.INT); - case BINARY_ARITHMETIC_EXPRESSION: - return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.INT), - generateExpression(depth + 1, CnosDBDataType.INT), - CnosDBBinaryOperator.getRandom(CnosDBDataType.INT)); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateUIntExpression(int depth) { - UIntExpression option = Randomly.fromOptions(UIntExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.UINT); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.UINT); - case BINARY_ARITHMETIC_EXPRESSION: - return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.UINT), - generateExpression(depth + 1, CnosDBDataType.UINT), - CnosDBBinaryOperator.getRandom(CnosDBDataType.UINT)); - default: - throw new AssertionError(); - } - - } - - private CnosDBExpression generateFloatExpression(int depth) { - FloatExpression option; - option = Randomly.fromOptions(FloatExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.DOUBLE); - case UNARY_OPERATION: - CnosDBExpression floatExpression = generateExpression(depth + 1, CnosDBDataType.DOUBLE); - return new CnosDBPrefixOperation(floatExpression, - Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.DOUBLE); - case BINARY_ARITHMETIC_EXPRESSION: - return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.DOUBLE), - generateExpression(depth + 1, CnosDBDataType.DOUBLE), - CnosDBBinaryOperator.getRandom(CnosDBDataType.DOUBLE)); - case CONSTANT: - return generateConstant(r, CnosDBDataType.DOUBLE); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateTimeStampExpression(int depth) { - if (Randomly.getBoolean()) { - return generateConstant(r, CnosDBDataType.TIMESTAMP); - } - TimestampExpression option; - option = Randomly.fromOptions(TimestampExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.TIMESTAMP); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.TIMESTAMP); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateCastExpression(int depth, CnosDBDataType castToType) { - CnosDBDataType castFromType = Randomly.fromList(CnosDBCastOperation.canCastTo(castToType)); - return new CnosDBCastOperation(generateExpression(depth + 1, castFromType), getCompoundDataType(castToType)); - } - - private CnosDBExpression createColumnOfType(CnosDBDataType type) { - List columns = filterColumns(type); - if (columns.isEmpty()) { - throw new IgnoreMeException(); - } - CnosDBColumn fromList = Randomly.fromList(columns); - return CnosDBColumnValue.create(fromList); - } - - final List filterColumns(CnosDBDataType type) { - if (columns == null) { - return Collections.emptyList(); - } else { - return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); - } - } - - public List generateExpressions(int nr) { - List expressions = new ArrayList<>(); - for (int i = 0; i < nr; i++) { - expressions.add(generateExpression(0)); - } - return expressions; - } - - public CnosDBExpression generateExpression(CnosDBDataType dataType) { - return generateExpression(0, dataType); - } - - public CnosDBExpression generateHavingClause() { - this.allowAggregateFunctions = true; - CnosDBExpression expression = generateExpression(CnosDBDataType.BOOLEAN); - this.allowAggregateFunctions = false; - return expression; - } - - public CnosDBExpression generateAggregate() { - return getAggregate(CnosDBDataType.getRandomType()); - } - - private CnosDBExpression getAggregate(CnosDBDataType dataType) { - if (dataType == CnosDBDataType.BOOLEAN) { - List aggregates = CnosDBAggregateFunction.getAggregates(CnosDBDataType.INT); - CnosDBAggregateFunction agg = Randomly.fromList(aggregates); - return new CnosDBCastOperation(generateArgsForAggregate(dataType, agg), - CnosDBCompoundDataType.create(CnosDBDataType.BOOLEAN)); - } else { - List aggregates = CnosDBAggregateFunction.getAggregates(dataType); - CnosDBAggregateFunction agg = Randomly.fromList(aggregates); - return generateArgsForAggregate(dataType, agg); - } - } - - public CnosDBAggregate generateArgsForAggregate(CnosDBDataType dataType, CnosDBAggregateFunction agg) { - CnosDBDataType[] types = agg.getArgsTypes(dataType); - List args = new ArrayList<>(); - for (CnosDBDataType argType : types) { - args.add(createColumnOfType(argType)); - // args.add(generateExpression(argType)); - } - return new CnosDBAggregate(args, agg); - } - - public CnosDBExpressionGenerator allowAggregates(boolean value) { - allowAggregateFunctions = value; - return this; - } - - @Override - public CnosDBExpression generatePredicate() { - return generateExpression(CnosDBDataType.BOOLEAN); - } - - @Override - public CnosDBExpression negatePredicate(CnosDBExpression predicate) { - return new CnosDBPrefixOperation(predicate, PrefixOperator.NOT); - } - - @Override - public CnosDBExpression isNull(CnosDBExpression expr) { - return new CnosDBPostfixOperation(expr, PostfixOperator.IS_NULL); - } - - private enum BooleanExpression { - POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, CAST, LIKE, BETWEEN, IN_OPERATION, - SIMILAR_TO, - } - - private enum StringExpression { - CAST, FUNCTION, CONCAT - } - - private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION - } - - private enum UIntExpression { - FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION - } - - private enum FloatExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, CONSTANT - } - - private enum TimestampExpression { - FUNCTION, CAST - } - -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java deleted file mode 100644 index 0d575d3c7..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java +++ /dev/null @@ -1,59 +0,0 @@ -package sqlancer.cnosdb.gen; - -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.schema.AbstractTableColumn; - -public final class CnosDBInsertGenerator { - - private CnosDBInsertGenerator() { - } - - public static CnosDBOtherQuery insert(CnosDBGlobalState globalState) { - CnosDBTable table = globalState.getSchema().getRandomTable(); - ExpectedErrors errors = new ExpectedErrors(); - errors.add("Column time cannot be null."); - StringBuilder sb = new StringBuilder(); - sb.append("INSERT "); - sb.append(table.getName()); - List columns = table.getRandomNonEmptyColumnSubset(); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES"); - - int n = Randomly.smallNumber() + 1; - for (int i = 0; i < n; i++) { - if (i != 0) { - sb.append(", "); - } - insertRow(globalState, sb, columns); - } - - // error - return new CnosDBOtherQuery(sb.toString(), errors); - } - - private static void insertRow(CnosDBGlobalState globalState, StringBuilder sb, List columns) { - sb.append("("); - for (int i = 0; i < columns.size(); i++) { - if (i > 0) { - sb.append(", "); - } - CnosDBExpression generateConstant = CnosDBExpressionGenerator.generateConstant(globalState.getRandomly(), - columns.get(i).getType()); - sb.append(CnosDBVisitor.asString(generateConstant)); - } - sb.append(")"); - } - -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java deleted file mode 100644 index c046ad3e9..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java +++ /dev/null @@ -1,77 +0,0 @@ -package sqlancer.cnosdb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBFieldColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTagColumn; -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.common.query.ExpectedErrors; - -public class CnosDBTableGenerator { - - protected final ExpectedErrors errors = new ExpectedErrors(); - private final String tableName; - private final StringBuilder sb = new StringBuilder(); - private final List columnsToBeAdd = new ArrayList<>(); - private CnosDBTable table; - - public CnosDBTableGenerator(String tableName) { - this.tableName = tableName; - } - - public static CnosDBOtherQuery generate(String tableName) { - return new CnosDBTableGenerator(tableName).generate(); - } - - protected CnosDBOtherQuery generate() { - table = new CnosDBTable(tableName, columnsToBeAdd); - - sb.append("CREATE TABLE"); - if (Randomly.getBoolean()) { - sb.append(" IF NOT EXISTS"); - } - sb.append(" "); - sb.append(tableName); - - sb.append("("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - String name = String.format("f%d", i); - createField(name); - sb.append(", "); - } - - sb.append("TAGS("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - if (i != 0) { - sb.append(", "); - } - String name = String.format("t%d", i); - createTag(name); - } - sb.append("))"); - return new CnosDBOtherQuery(sb.toString(), new ExpectedErrors()); - } - - private void createField(String name) throws AssertionError { - sb.append(name); - sb.append(" "); - CnosDBDataType type = CnosDBDataType.getRandomTypeWithoutTimeStamp(); - CnosDBCommon.appendDataType(type, sb); - CnosDBFieldColumn c = new CnosDBFieldColumn(name, type); - c.setTable(table); - sb.append(" "); - columnsToBeAdd.add(c); - } - - private void createTag(String name) { - sb.append(name); - CnosDBColumn column = new CnosDBTagColumn(name); - column.setTable(table); - columnsToBeAdd.add(column); - } -} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java deleted file mode 100644 index 472aa8f66..000000000 --- a/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java +++ /dev/null @@ -1,23 +0,0 @@ -package sqlancer.cnosdb.oracle; - -import sqlancer.Main; -import sqlancer.MainOptions; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.oracle.TestOracle; - -public abstract class CnosDBNoRECBase implements TestOracle { - protected final CnosDBGlobalState state; - protected final Main.StateLogger logger; - protected final MainOptions options; - protected final CnosDBConnection con; - protected String optimizedQueryString; - protected String unoptimizedQueryString; - - public CnosDBNoRECBase(CnosDBGlobalState state) { - this.state = state; - this.con = state.getConnection(); - this.logger = state.getLogger(); - this.options = state.getOptions(); - } -} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java deleted file mode 100644 index 0c817c655..000000000 --- a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java +++ /dev/null @@ -1,171 +0,0 @@ -package sqlancer.cnosdb.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBCompoundDataType; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBJoin.CnosDBJoinType; -import sqlancer.cnosdb.ast.CnosDBPostfixText; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.ast.CnosDBSelect.SelectType; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPBase; -import sqlancer.cnosdb.query.CnosDBSelectQuery; -import sqlancer.common.oracle.TestOracle; - -public class CnosDBNoRECOracle extends CnosDBNoRECBase implements TestOracle { - - private final CnosDBSchema s; - - public CnosDBNoRECOracle(CnosDBGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - } - - public static List getJoinStatements(CnosDBGlobalState globalState, List columns, - List tables) { - List joinStatements = new ArrayList<>(); - CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); - CnosDBTable table = Randomly.fromList(tables); - tables.remove(table); - CnosDBJoinType options = CnosDBJoinType.getRandom(); - CnosDBJoin j = new CnosDBJoin(new CnosDBFromTable(table), joinClause, options); - joinStatements.add(j); - } - // JOIN subqueries - for (int i = 0; i < Randomly.smallNumber(); i++) { - CnosDBTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); - CnosDBSubquery subquery = CnosDBTLPBase.createSubquery(globalState, String.format("sub%d", i), - subqueryTables); - CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); - CnosDBJoinType options = CnosDBJoinType.getRandom(); - CnosDBJoin j = new CnosDBJoin(subquery, joinClause, options); - joinStatements.add(j); - } - return joinStatements; - } - - @Override - public void check() throws Exception { - CnosDBTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - CnosDBExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); - - List joinStatements = getJoinStatements(state, columns, tables); - List fromTables = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, List.of(CnosDBColumn.createDummy("f0")), - randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } - } - - private CnosDBExpression getRandomWhereCondition(List columns) { - return new CnosDBExpressionGenerator(state).setColumns(columns).generateExpression(CnosDBDataType.BOOLEAN); - } - - private int getUnoptimizedQueryCount(List fromTables, CnosDBExpression randomWhereCondition, - List joinStatements) throws Exception { - CnosDBSelect select = new CnosDBSelect(); - CnosDBCastOperation isTrue = new CnosDBCastOperation(randomWhereCondition, - CnosDBCompoundDataType.create(CnosDBDataType.INT)); - CnosDBPostfixText asText = new CnosDBPostfixText(isTrue, " as count", CnosDBDataType.INT); - select.setFetchColumns(List.of(asText)); - select.setFromList(fromTables); - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + CnosDBVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - CnosDBSelectQuery q = new CnosDBSelectQuery(unoptimizedQueryString, CnosDBExpectedError.expectedErrors()); - CnosDBResultSet rs; - try { - q.executeAndGet(state); - rs = q.getResultSet(); - } catch (Exception e) { - if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(List randomTables, List columns, - CnosDBExpression randomWhereCondition, List joinStatements) { - CnosDBSelect select = new CnosDBSelect(); - CnosDBColumnValue allColumns = new CnosDBColumnValue(Randomly.fromList(columns)); - select.setFetchColumns(List.of(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new CnosDBExpressionGenerator(state).setColumns(columns).generateOrderBy()); - } - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - optimizedQueryString = CnosDBVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - CnosDBSelectQuery query = new CnosDBSelectQuery(optimizedQueryString, CnosDBExpectedError.expectedErrors()); - CnosDBResultSet rs; - try { - query.executeAndGet(state); - rs = query.getResultSet(); - while (rs.next()) { - firstCount++; - } - } catch (Exception e) { - if (query.getExpectedErrors().errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - - throw new IgnoreMeException(); - } - return firstCount; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java deleted file mode 100644 index b51624a94..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java +++ /dev/null @@ -1,176 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import sqlancer.ComparatorHelper; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; -import sqlancer.cnosdb.ast.CnosDBAlias; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.cnosdb.query.CnosDBSelectQuery; -import sqlancer.common.oracle.TestOracle; - -public class CnosDBTLPAggregateOracle extends CnosDBTLPBase implements TestOracle { - - private String firstResult; - private String secondResult; - private String originalQuery; - private String metamorphicQuery; - - public CnosDBTLPAggregateOracle(CnosDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - aggregateCheck(); - } - - protected void aggregateCheck() { - CnosDBAggregateFunction aggregateFunction = Randomly.fromOptions(CnosDBAggregateFunction.MAX, - CnosDBAggregateFunction.MIN, CnosDBAggregateFunction.SUM); - - CnosDBAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), - aggregateFunction); - List fetchColumns = new ArrayList<>(); - fetchColumns.add(aggregate); - while (Randomly.getBooleanWithRatherLowProbability()) { - fetchColumns.add(gen.generateAggregate()); - } - select.setFetchColumns(fetchColumns); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - originalQuery = CnosDBVisitor.asString(select); - firstResult = getAggregateResult(originalQuery); - metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); - secondResult = getAggregateResult(metamorphicQuery); - - String queryFormatString = "-- %s;\n-- result: %s"; - String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); - String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - if (firstResult == null && secondResult != null || firstResult != null && secondResult == null - || firstResult != null && !firstResult.contentEquals(secondResult) - && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { - if (secondResult != null && secondResult.contains("Inf")) { - throw new IgnoreMeException(); // FIXME: average computation - } - String assertionMessage = String.format("%s: the results mismatch!\n%s\n%s", this.s.getDatabaseName(), - firstQueryString, secondQueryString); - throw new AssertionError(assertionMessage); - } - } - - private String createMetamorphicUnionQuery(CnosDBSelect select, CnosDBAggregate aggregate, - List from) { - String metamorphicQuery; - CnosDBExpression whereClause = gen.generateExpression(CnosDBDataType.BOOLEAN); - CnosDBExpression negatedClause = new CnosDBPrefixOperation(whereClause, PrefixOperator.NOT); - CnosDBExpression notNullClause = new CnosDBPostfixOperation(whereClause, PostfixOperator.IS_NULL); - List mappedAggregate = mapped(aggregate); - CnosDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); - CnosDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); - CnosDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; - metamorphicQuery += CnosDBVisitor.asString(leftSelect) + " UNION ALL " + CnosDBVisitor.asString(middleSelect) - + " UNION ALL " + CnosDBVisitor.asString(rightSelect); - metamorphicQuery += ") as asdf"; - return metamorphicQuery; - } - - private String getAggregateResult(String queryString) { - // log TLP Aggregate SELECT queries on the current log file - if (state.getOptions().logEachSelect()) { - // TODO: refactor me - state.getLogger().writeCurrent(queryString); - try { - state.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - String resultString = null; - - CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, CnosDBExpectedError.expectedErrors()); - try { - q.executeAndGet(state); - CnosDBResultSet result = q.getResultSet(); - - if (result == null || !result.next()) { - throw new IgnoreMeException(); - } - - resultString = result.getString(1); - - } catch (Exception e) { - if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - } - - return resultString; - } - - private List mapped(CnosDBAggregate aggregate) { - switch (aggregate.getFunction()) { - case SUM: - case MAX: - case MIN: - return aliasArgs(List.of(aggregate)); - // now not support - // case COUNT: - // case AVG: - default: - throw new AssertionError(aggregate.getFunction()); - } - } - - private List aliasArgs(List originalAggregateArgs) { - List args = new ArrayList<>(); - int i = 0; - for (CnosDBExpression expr : originalAggregateArgs) { - args.add(new CnosDBAlias(expr, "agg" + i++)); - } - return args; - } - - private String getOuterAggregateFunction(CnosDBAggregate aggregate) { - if (Objects.requireNonNull(aggregate.getFunction()) == CnosDBAggregateFunction.COUNT) { - return CnosDBAggregateFunction.SUM + "(agg0)"; - } - return aggregate.getFunction() + "(agg0)"; - } - - private CnosDBSelect getSelect(List aggregates, List from, - CnosDBExpression whereClause, List joinList) { - CnosDBSelect leftSelect = new CnosDBSelect(); - leftSelect.setFetchColumns(aggregates); - leftSelect.setFromList(from); - leftSelect.setWhereClause(whereClause); - leftSelect.setJoinClauses(joinList); - if (Randomly.getBooleanWithSmallProbability()) { - leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - } - return leftSelect; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java deleted file mode 100644 index bd7ba3b55..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java +++ /dev/null @@ -1,112 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; -import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; - -public class CnosDBTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - protected CnosDBSchema s; - protected CnosDBTables targetTables; - protected CnosDBExpressionGenerator gen; - protected CnosDBSelect select; - - public CnosDBTLPBase(CnosDBGlobalState state) { - super(state); - } - - public static CnosDBSubquery createSubquery(CnosDBGlobalState globalState, String name, CnosDBTables tables) { - List columns = new ArrayList<>(); - CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(tables.getColumns()); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - columns.add(gen.generateExpression(0)); - } - CnosDBSelect select = new CnosDBSelect(); - select.setFromList(tables.getTables().stream().map(CnosDBFromTable::new).collect(Collectors.toList())); - select.setFetchColumns(columns); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(0, CnosDBDataType.BOOLEAN)); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - if (Randomly.getBoolean()) { - select.setLimitClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - if (Randomly.getBoolean()) { - select.setOffsetClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - } - } - return new CnosDBSubquery(select, name); - } - - @Override - public void check() throws Exception { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - List tables = targetTables.getTables(); - List joins = getJoinStatements(targetTables.getColumns(), tables); - generateSelectBase(tables, joins); - } - - protected List getJoinStatements(List columns, List tables) { - return CnosDBNoRECOracle.getJoinStatements(state, columns, tables); - } - - protected void generateSelectBase(List tables, List joins) { - List tableList = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); - gen = new CnosDBExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new CnosDBSelect(); - select.setFetchColumns(generateFetchColumns()); - select.setFromList(tableList); - select.setWhereClause(null); - select.setJoinClauses(joins); - } - - List generateFetchColumns() { - if (Randomly.getBooleanWithRatherLowProbability()) { - return List.of(new CnosDBColumnValue(CnosDBColumn.createDummy("*"))); - } - List fetchColumns = new ArrayList<>(); - List targetColumns = targetTables.getRandomColumnsWithOnlyOneField(); - - ArrayList columns = new ArrayList<>(); - targetColumns.forEach(column -> column.getTable().getColumns().stream() - .filter(field -> field instanceof CnosDBSchema.CnosDBFieldColumn).findFirst().ifPresent(columns::add)); - targetColumns.addAll(columns); - - targetColumns = targetColumns.stream().distinct().collect(Collectors.toList()); - - for (CnosDBColumn c : targetColumns) { - fetchColumns.add(new CnosDBColumnValue(c)); - } - return fetchColumns; - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java deleted file mode 100644 index 283d59a23..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java +++ /dev/null @@ -1,65 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBComparatorHelper; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBExpression; - -public class CnosDBTLPHavingOracle extends CnosDBTLPBase { - - public CnosDBTLPHavingOracle(CnosDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - havingCheck(); - } - - protected void havingCheck() throws Exception { - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(CnosDBDataType.BOOLEAN)); - } - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - select.setHavingClause(null); - String originalQueryString = CnosDBVisitor.asString(select); - List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, - CnosDBExpectedError.expectedErrors(), state); - - boolean orderBy = Randomly.getBoolean(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBy()); - } - select.setHavingClause(predicate); - String firstQueryString = CnosDBVisitor.asString(select); - select.setHavingClause(negatedPredicate); - String secondQueryString = CnosDBVisitor.asString(select); - select.setHavingClause(isNullPredicate); - String thirdQueryString = CnosDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, CnosDBExpectedError.expectedErrors()); - CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - - @Override - protected CnosDBExpression generatePredicate() { - return gen.generateHavingClause(); - } - - @Override - List generateFetchColumns() { - List expressions = gen.allowAggregates(true).generateExpressions(Randomly.smallNumber() + 1); - gen.allowAggregates(false); - return expressions; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java deleted file mode 100644 index 8e118435d..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBComparatorHelper; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBVisitor; - -public class CnosDBTLPWhereOracle extends CnosDBTLPBase { - - public CnosDBTLPWhereOracle(CnosDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - whereCheck(); - } - - protected void whereCheck() throws Exception { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - String originalQueryString = CnosDBVisitor.asString(select); - List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, - CnosDBExpectedError.expectedErrors(), state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = CnosDBVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = CnosDBVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = CnosDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, CnosDBExpectedError.expectedErrors()); - CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } -} diff --git a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java deleted file mode 100644 index f0a37056c..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java +++ /dev/null @@ -1,32 +0,0 @@ -package sqlancer.cnosdb.query; - -import sqlancer.GlobalState; -import sqlancer.IgnoreMeException; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.query.ExpectedErrors; - -public class CnosDBOtherQuery extends CnosDBQueryAdapter { - private static final long serialVersionUID = 1L; - - public CnosDBOtherQuery(String query, ExpectedErrors errors) { - super(query, errors); - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - globalState.getConnection().getClient().execute(query); - } catch (Exception e) { - if (this.errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - } - return true; - } -} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java deleted file mode 100644 index 115f96ffc..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.cnosdb.query; - -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; - -public abstract class CnosDBQueryAdapter extends Query { - private static final long serialVersionUID = 1L; - - String query; - ExpectedErrors errors; - - public CnosDBQueryAdapter(String query, ExpectedErrors errors) { - this.query = query; - this.errors = errors; - } - - @Override - public String getLogString() { - return query; - } - - @Override - public String getQueryString() { - return query; - } - - @Override - public String getUnterminatedQueryString() { - return null; - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public ExpectedErrors getExpectedErrors() { - return errors; - } -} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java b/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java deleted file mode 100644 index dee38abf4..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.cnosdb.query; - -@FunctionalInterface -public interface CnosDBQueryProvider { - CnosDBOtherQuery getQuery(S globalState) throws Exception; -} diff --git a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java deleted file mode 100644 index 1c9228182..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.cnosdb.query; - -import sqlancer.GlobalState; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLancerResultSet; - -public class CnosDBSelectQuery extends CnosDBQueryAdapter { - private static final long serialVersionUID = 1L; - CnosDBResultSet resultSet; - - public CnosDBSelectQuery(String query, ExpectedErrors errors) { - super(query, errors); - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - globalState.getConnection().getClient().execute(query); - return false; - } - - @Override - public > SQLancerResultSet executeAndGet(G globalState, - String... fills) throws Exception { - resultSet = globalState.getConnection().getClient().executeQuery(query); - return null; - } - - public CnosDBResultSet getResultSet() { - return resultSet; - } -} diff --git a/test/sqlancer/dbms/TestCnosDBNoREC.java b/test/sqlancer/dbms/TestCnosDBNoREC.java deleted file mode 100644 index 1a89a972a..000000000 --- a/test/sqlancer/dbms/TestCnosDBNoREC.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestCnosDBNoREC { - - @Test - public void testCnosDBNoREC() { - assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); - // Run with 0 queries as current implementation is resulting in database crashes - assertEquals(0, - Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", - "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", - "--oracle", "NOREC" })); - } - -} diff --git a/test/sqlancer/dbms/TestCnosDBTLP.java b/test/sqlancer/dbms/TestCnosDBTLP.java deleted file mode 100644 index 4b12aa409..000000000 --- a/test/sqlancer/dbms/TestCnosDBTLP.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestCnosDBTLP { - - @Test - public void testCnosDBTLP() { - assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); - // Run with 0 queries as current implementation is resulting in database crashes - assertEquals(0, - Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", - "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", - "--oracle", "QUERY_PARTITIONING" })); - } - -} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index f6be45648..f2372f266 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -5,7 +5,6 @@ public class TestConfig { public static final String SECONDS = "300"; public static final String CLICKHOUSE_ENV = "CLICKHOUSE_AVAILABLE"; - public static final String CNOSDB_ENV = "CNOSDB_AVAILABLE"; public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; From 7ba8bba56c5397d6b6d345add31ac9be420b4a41 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 23:13:37 +0800 Subject: [PATCH 071/132] Remove Databend bug19738 workaround after upstream fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue (databendlabs/databend#19738) was fixed in databendlabs/databend#19740 — re-enable AVG in aggregate testing and stop suppressing the related error messages. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/DatabendBugs.java | 1 - src/sqlancer/databend/DatabendErrors.java | 4 ---- .../tlp/DatabendQueryPartitioningAggregateTester.java | 11 ++++------- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e85c5c219..437e40350 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.896-nightly + image: datafuselabs/databend:v1.2.900-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index a058ccc32..dd11512d8 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -19,7 +19,6 @@ public final class DatabendBugs { public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 - public static boolean bug19738 = true; // https://github.com/databendlabs/databend/issues/19738 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 746a4e848..fdd8a3a69 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -47,10 +47,6 @@ public static List getExpressionErrors() { if (DatabendBugs.bug15568) { errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); } - if (DatabendBugs.bug19738) { - errors.add("UnwindError"); - errors.add("unable to cast `NULL`"); - } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 6d52caea3..ee3656413 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -10,7 +10,6 @@ import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -45,12 +44,10 @@ public DatabendQueryPartitioningAggregateTester(DatabendGlobalState state) { @Override public void check() throws SQLException { super.check(); - List aggregateFunctions = new ArrayList<>(List.of(DatabendAggregateFunction.MAX, - DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT - /* , DatabendAggregateFunction.STDDEV_POP */)); - if (!DatabendBugs.bug19738) { - aggregateFunctions.add(DatabendAggregateFunction.AVG); - } + List aggregateFunctions = new ArrayList<>( + List.of(DatabendAggregateFunction.MAX, DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, + DatabendAggregateFunction.COUNT, DatabendAggregateFunction.AVG + /* , DatabendAggregateFunction.STDDEV_POP */)); DatabendAggregateFunction aggregateFunction = Randomly.fromList(aggregateFunctions); DatabendFunctionOperation aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); From cbd284789221a6ee5680ae3c37753fec9cb815e0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 23:15:16 +0800 Subject: [PATCH 072/132] Suppress Databend bug19773 (eager-aggregation Decimal mismatch family) Filed as databendlabs/databend#19773. The narrow AVG-only fix in databendlabs/databend#19740 left several related shapes still broken on v1.2.900-nightly, all in the eager-aggregation rewrite path: 1. SUM(decimal_literal) over a cross join inside UNION ALL with an outer aggregate fails with `failed to downcast column Decimal128(...) into ... CoreDecimal`. 2. Plain SUM(decimal_literal) over an N-table cross join fails with `assertion left == right ... Decimal precision: 38 vs 18` once the SUM result is wide enough to be promoted to Decimal128. The original report (#19738) claimed SUM did not trigger the bug; it does, just at higher cardinalities than AVG does. 3. Outer SUM over UNION ALL of inner COUNTs fails with `unable to cast `NULL` to type `UInt64` ... CAST(_eager_final_count (#N) AS UInt64)`. Suppress all three narrowly via distinctive substrings rather than reinstating the broad `UnwindError` / `unable to cast `NULL`` matches the previous commit removed. Co-Authored-By: Claude Opus 4.7 --- src/sqlancer/databend/DatabendBugs.java | 1 + src/sqlancer/databend/DatabendErrors.java | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index dd11512d8..ae2f74a33 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -19,6 +19,7 @@ public final class DatabendBugs { public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 + public static boolean bug19773 = true; // https://github.com/databendlabs/databend/issues/19773 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index fdd8a3a69..3e056d003 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -47,6 +47,11 @@ public static List getExpressionErrors() { if (DatabendBugs.bug15568) { errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); } + if (DatabendBugs.bug19773) { + errors.add("failed to downcast column Decimal128"); + errors.add("Decimal(DecimalSize { precision: 38"); + errors.add("_eager_final_count"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE From da231dd6c28716ec5af893372be7ab6c8f80e7b1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 23:55:28 +0800 Subject: [PATCH 073/132] Citus: skip INHERITS generation until citusdata/citus#8553 is fixed Citus's distributed planner returns wrong results when an inheritance parent is cross-joined with a distributed table inside a LEFT JOIN ... ON FALSE and a WHERE filters on the parent column, which the TLP-WHERE oracle keeps tripping over (~1 in 8 Citus CI runs). Make PostgresTableGenerator.generateInherits() protected so CitusTableGenerator can override it as a no-op while the new CitusBugs.bug8553 flag is set. Co-Authored-By: Claude Opus 4.7 --- src/sqlancer/citus/CitusBugs.java | 3 +++ src/sqlancer/citus/gen/CitusTableGenerator.java | 9 +++++++++ src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/citus/CitusBugs.java b/src/sqlancer/citus/CitusBugs.java index a6f4910e6..1fda3be6a 100644 --- a/src/sqlancer/citus/CitusBugs.java +++ b/src/sqlancer/citus/CitusBugs.java @@ -33,6 +33,9 @@ public final class CitusBugs { // https://github.com/citusdata/citus/issues/6298 public static boolean bug6298 = true; + // https://github.com/citusdata/citus/issues/8553 + public static boolean bug8553 = true; + private CitusBugs() { } diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java index 86e5d40cf..5d6b8b249 100644 --- a/src/sqlancer/citus/gen/CitusTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -1,5 +1,6 @@ package sqlancer.citus.gen; +import sqlancer.citus.CitusBugs; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; @@ -20,4 +21,12 @@ public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchem return new CitusTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); } + @Override + protected void generateInherits() { + if (CitusBugs.bug8553) { + return; + } + super.generateInherits(); + } + } diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 29ccfcf2c..9a7bfb032 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -206,7 +206,7 @@ private void generateUsing() { sb.append(globalState.getRandomTableAccessMethod()); } - private void generateInherits() { + protected void generateInherits() { if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) From d2b7151a96c3a482a221969ded411d7ef1dd4cfa Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 30 May 2026 16:39:08 +0100 Subject: [PATCH 074/132] Update MySQL CI to MySQL 9.7.0 and JDBC driver to mysql-connector-j 9.7.0 --- .github/workflows/main.yml | 2 +- pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c511fc0a..4543900b7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -444,7 +444,7 @@ jobs: runs-on: ubuntu-latest services: mysql: - image: mysql:8.4 + image: mysql:9.7.0 env: MYSQL_ROOT_PASSWORD: root ports: diff --git a/pom.xml b/pom.xml index 7c9a1106b..c4bc71f82 100644 --- a/pom.xml +++ b/pom.xml @@ -302,9 +302,9 @@ 3.49.1.0 - mysql - mysql-connector-java - 8.0.30 + com.mysql + mysql-connector-j + 9.7.0 org.mariadb.jdbc From 3efe71a4f193fed29a7815ac9d020e1c83f7f6b8 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 30 May 2026 16:46:17 +0100 Subject: [PATCH 075/132] Add 'incorrect FLOAT value' as expected error following update to MySQL 9.7.0 --- src/sqlancer/mysql/MySQLErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 989c8fed6..6843a27c3 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -49,6 +49,7 @@ public static List getInsertUpdateErrors() { errors.add("doesn't have a default value"); errors.add("Data truncation"); errors.add("Incorrect integer value"); + errors.add("Incorrect FLOAT value"); errors.add("Duplicate entry"); errors.add("Data truncated for column"); errors.add("Data truncated for functional index"); From 725d31de0e2702ecf7d09a6d4e3417d57493f849 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 30 May 2026 16:49:36 +0100 Subject: [PATCH 076/132] Add 'incorrect DOUBLE value' as expected error following update to MySQL 9.7.0 --- src/sqlancer/mysql/MySQLErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 6843a27c3..cb9ad4f01 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -50,6 +50,7 @@ public static List getInsertUpdateErrors() { errors.add("Data truncation"); errors.add("Incorrect integer value"); errors.add("Incorrect FLOAT value"); + errors.add("Incorrect DOUBLE value"); errors.add("Duplicate entry"); errors.add("Data truncated for column"); errors.add("Data truncated for functional index"); From 9dbefd9269463471c5fd3d6199fc182ce993ccfd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 1 Jun 2026 17:32:24 +0100 Subject: [PATCH 077/132] Implement workaround for MySQL CREATE INDEX on integer column bug --- src/sqlancer/mysql/MySQLBugs.java | 4 +++ src/sqlancer/mysql/ast/MySQLConstant.java | 9 +++++++ .../mysql/gen/MySQLInsertGenerator.java | 27 +++++++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 8cb8a3391..1f9b7ce2f 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -37,6 +37,10 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=114534 public static boolean bug114534 = true; + // https://bugs.mysql.com/bug.php?id=120711 + // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. + public static boolean bug120711 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index 2e4922f8e..5fb0698b9 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -68,6 +68,11 @@ public MySQLDoubleConstant(double val) { } } + @Override + public double getDouble() { + return val; + } + @Override public String getTextRepresentation() { return String.valueOf(val); @@ -381,6 +386,10 @@ public long getInt() { throw new UnsupportedOperationException(); } + public double getDouble() { + throw new UnsupportedOperationException(); + } + public boolean isSigned() { return false; } diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 86083fd2d..696231a24 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -8,10 +8,14 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLConstant; public class MySQLInsertGenerator { @@ -84,8 +88,27 @@ private SQLQueryAdapter generateInto() { if (c != 0) { sb.append(", "); } - sb.append(MySQLVisitor.asString(gen.generateConstant())); - + MySQLExpression constExpr; + // Bug workaround: for integer columns, reject numeric values that round to 1. Regenerate until valid. + if (MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { + while (true) { + constExpr = gen.generateConstant(); + boolean reject = false; + if (constExpr instanceof MySQLConstant.MySQLIntConstant) { + long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); + reject = value == 1; + } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { + double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); + reject = value >= 0.5 && value < 1.5; + } + if (!reject) { + break; + } + } + } else { + constExpr = gen.generateConstant(); + } + sb.append(MySQLVisitor.asString(constExpr)); } sb.append(")"); } From 2112337c335b2b01e8d6d50a978e853053e70771 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 1 Jun 2026 11:05:16 +0100 Subject: [PATCH 078/132] Implement workaround for MySQL DECIMAL UNIQUE bug --- src/sqlancer/mysql/MySQLBugs.java | 4 ++++ src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 1f9b7ce2f..9f0b7589c 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -37,6 +37,10 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=114534 public static boolean bug114534 = true; + // https://bugs.mysql.com/bug.php?id=120710 + // Inserting a NULL and a value which rounds to 0 into a DECIMAL column causes result set mismatch. + public static boolean bug120710 = true; + // https://bugs.mysql.com/bug.php?id=120711 // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. public static boolean bug120711 = true; diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 696231a24..dc2b51bf7 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -105,6 +105,22 @@ private SQLQueryAdapter generateInto() { break; } } + // Bug workaround: for decimal columns, reject values that round to 0. Regenerate until valid. + } else if (MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { + while (true) { + constExpr = gen.generateConstant(); + boolean reject = false; + if (constExpr instanceof MySQLConstant.MySQLIntConstant) { + long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); + reject = value == 0; + } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { + double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); + reject = value >= -0.5 && value < 0.5; + } + if (!reject) { + break; + } + } } else { constExpr = gen.generateConstant(); } From f857438c035761c63836971503847ed8212c16b1 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 8 Jun 2026 10:22:30 +0800 Subject: [PATCH 079/132] Make the workaround for MySQL insertion bugs more robust against string generation (which may implicitly cast to undesirable integer/double) --- src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index dc2b51bf7..b57ab5158 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -100,6 +100,8 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= 0.5 && value < 1.5; + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 1 + reject = true; } if (!reject) { break; @@ -116,6 +118,8 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= -0.5 && value < 0.5; + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 0 + reject = true; } if (!reject) { break; From 04b7549e04bdc693367086fda6196ddf5645b20c Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 8 Jun 2026 15:10:18 +0800 Subject: [PATCH 080/132] Fix EXPLAIN format following change of default since previous version of MySQL --- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 98641ab26..8af6923e0 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -253,7 +253,7 @@ public List generateFetchColumns(boolean shouldCreateDummy) { @Override public String generateExplainQuery(MySQLSelect select) { - return "EXPLAIN " + select.asString(); + return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed from TRADITIONAL to TREE, hence TRADITIONAL must now be specified } public MySQLAggregate generateAggregate() { From e004455565157ce630499346c20cbe9a2351b100 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 9 Jun 2026 14:40:27 +0800 Subject: [PATCH 081/132] Refactor workaround logic for MySQL insertion bugs for easier extension --- .../mysql/gen/MySQLInsertGenerator.java | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index b57ab5158..d15d9def1 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -89,11 +89,13 @@ private SQLQueryAdapter generateInto() { sb.append(", "); } MySQLExpression constExpr; - // Bug workaround: for integer columns, reject numeric values that round to 1. Regenerate until valid. - if (MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { - while (true) { - constExpr = gen.generateConstant(); - boolean reject = false; + // loop to regenerate until expression is valid (for bug workarounds) + while (true) { + constExpr = gen.generateConstant(); + boolean reject = false; + + // Bug workaround: for integer columns, reject values that round to 1 + if (!reject && MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { if (constExpr instanceof MySQLConstant.MySQLIntConstant) { long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); reject = value == 1; @@ -103,15 +105,10 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 1 reject = true; } - if (!reject) { - break; - } } - // Bug workaround: for decimal columns, reject values that round to 0. Regenerate until valid. - } else if (MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { - while (true) { - constExpr = gen.generateConstant(); - boolean reject = false; + + // Bug workaround: for decimal columns, reject values that round to 0 + if (!reject && MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { if (constExpr instanceof MySQLConstant.MySQLIntConstant) { long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); reject = value == 0; @@ -121,12 +118,11 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 0 reject = true; } - if (!reject) { - break; - } } - } else { - constExpr = gen.generateConstant(); + + if (!reject) { + break; + } } sb.append(MySQLVisitor.asString(constExpr)); } From 383d79c8de0f72fc49ce59a5276ea1a023827561 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 9 Jun 2026 15:21:24 +0800 Subject: [PATCH 082/132] Implement workaround for MySQL CREATE INDEX between NULL inserts CERT bug --- src/sqlancer/mysql/MySQLBugs.java | 4 ++++ src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 9f0b7589c..8032d78ab 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -45,6 +45,10 @@ public final class MySQLBugs { // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. public static boolean bug120711 = true; + // https://bugs.mysql.com/bug.php?id=120712 + // Creating an index in between two NULL inserts causes inconsistent CERT result. + public static boolean bug120712 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index d15d9def1..a0f44ac9e 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -10,6 +10,7 @@ import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -120,6 +121,13 @@ private SQLQueryAdapter generateInto() { } } + // Bug workaround: if using CERT oracle, reject NULL values + if (!reject && MySQLBugs.bug120712 && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.CERT)) { + if (constExpr instanceof MySQLConstant.MySQLNullConstant) { + reject = true; + } + } + if (!reject) { break; } From 2eced5c4c33b321105d8d9c17ff2069228f1399e Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 12:48:38 +0800 Subject: [PATCH 083/132] Implement workaround for oracles reporting inconsistency with ZEROFILL in MySQL despite the behaviour being expected --- src/sqlancer/mysql/MySQLBugs.java | 6 +----- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 2 +- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 5 +++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 8032d78ab..5ad2cfcbd 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -3,12 +3,8 @@ // do not make the fields final to avoid warnings public final class MySQLBugs { - // https://bugs.mysql.com/bug.php?id=99127 0.9 > t0.c0 malfunctions when c0 is - // an INT UNSIGNED - public static boolean bug99127 = true; - // https://bugs.mysql.com/99182 BETWEEN malfunctions for DECIMAL and TEXT - public static boolean bug99181 = true; + public static boolean bug99182 = true; // https://bugs.mysql.com/bug.php?id=99183 public static boolean bug99183 = true; diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 8af6923e0..513867e39 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -112,7 +112,7 @@ public MySQLExpression generateExpression(int depth) { case EXISTS: return getExists(); case BETWEEN_OPERATOR: - if (MySQLBugs.bug99181) { + if (MySQLBugs.bug99182) { // TODO: there are a number of bugs that are triggered by the BETWEEN operator throw new IgnoreMeException(); } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index bc0533295..17e5e9f13 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -13,6 +13,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; @@ -356,10 +357,10 @@ private void appendType(MySQLDataType randomType) { throw new AssertionError(); } if (randomType.isNumeric()) { - if (Randomly.getBoolean() && randomType != MySQLDataType.INT && !MySQLBugs.bug99127) { + if (Randomly.getBoolean() && randomType != MySQLDataType.INT) { sb.append(" UNSIGNED"); } - if (!globalState.usesPQS() && Randomly.getBoolean()) { + if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS || o == MySQLOracleFactory.DQP)) { sb.append(" ZEROFILL"); } } From 4fad414a842a346b788add59608c0a523ef1a737 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 18:24:33 +0800 Subject: [PATCH 084/132] Add loop counter for MySQLInsertGenerator regeneration attempts --- src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index a0f44ac9e..a35549405 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -24,6 +24,7 @@ public class MySQLInsertGenerator { private final StringBuilder sb = new StringBuilder(); private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; + private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug workarounds) public MySQLInsertGenerator(MySQLGlobalState globalState, MySQLTable table) { this.globalState = globalState; @@ -91,7 +92,12 @@ private SQLQueryAdapter generateInto() { } MySQLExpression constExpr; // loop to regenerate until expression is valid (for bug workarounds) + int regenerationAttempts = 0; while (true) { + regenerationAttempts++; + if (regenerationAttempts > MAX_REGENERATION_ATTEMPTS) { + throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + " attempts while generating constant for column " + columns.get(c).getName()); + } constExpr = gen.generateConstant(); boolean reject = false; From 5004bc5730e72a9f380de73c4437d3d7dcf76439 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 18:36:32 +0800 Subject: [PATCH 085/132] Format to pass CI tests --- src/sqlancer/mysql/MySQLBugs.java | 3 ++- .../mysql/gen/MySQLExpressionGenerator.java | 4 ++- .../mysql/gen/MySQLInsertGenerator.java | 25 +++++++++++-------- .../mysql/gen/MySQLTableGenerator.java | 4 ++- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 5ad2cfcbd..e4fae3cd7 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -38,7 +38,8 @@ public final class MySQLBugs { public static boolean bug120710 = true; // https://bugs.mysql.com/bug.php?id=120711 - // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. + // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set + // mismatch. public static boolean bug120711 = true; // https://bugs.mysql.com/bug.php?id=120712 diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 513867e39..d8ce5dd37 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -253,7 +253,9 @@ public List generateFetchColumns(boolean shouldCreateDummy) { @Override public String generateExplainQuery(MySQLSelect select) { - return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed from TRADITIONAL to TREE, hence TRADITIONAL must now be specified + return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed + // from TRADITIONAL to TREE, hence TRADITIONAL must + // now be specified } public MySQLAggregate generateAggregate() { diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index a35549405..0e464dead 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -7,16 +7,16 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLBugs; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; -import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExpression; public class MySQLInsertGenerator { @@ -24,7 +24,8 @@ public class MySQLInsertGenerator { private final StringBuilder sb = new StringBuilder(); private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; - private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug workarounds) + private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug + // workarounds) public MySQLInsertGenerator(MySQLGlobalState globalState, MySQLTable table) { this.globalState = globalState; @@ -96,7 +97,8 @@ private SQLQueryAdapter generateInto() { while (true) { regenerationAttempts++; if (regenerationAttempts > MAX_REGENERATION_ATTEMPTS) { - throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + " attempts while generating constant for column " + columns.get(c).getName()); + throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + + " attempts while generating constant for column " + columns.get(c).getName()); } constExpr = gen.generateConstant(); boolean reject = false; @@ -109,7 +111,8 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= 0.5 && value < 1.5; - } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 1 + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may + // be implicitly cast to 1 reject = true; } } @@ -122,16 +125,18 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= -0.5 && value < 0.5; - } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 0 + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may + // be implicitly cast to 0 reject = true; } } // Bug workaround: if using CERT oracle, reject NULL values - if (!reject && MySQLBugs.bug120712 && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.CERT)) { - if (constExpr instanceof MySQLConstant.MySQLNullConstant) { - reject = true; - } + if (!reject && MySQLBugs.bug120712 + && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.CERT) + && constExpr instanceof MySQLConstant.MySQLNullConstant) { + reject = true; } if (!reject) { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 17e5e9f13..054a66cb6 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -360,7 +360,9 @@ private void appendType(MySQLDataType randomType) { if (Randomly.getBoolean() && randomType != MySQLDataType.INT) { sb.append(" UNSIGNED"); } - if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS || o == MySQLOracleFactory.DQP)) { + if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS + || o == MySQLOracleFactory.DQP)) { sb.append(" ZEROFILL"); } } From df5f32ecc1f2d52b675d5534ea68880d6439167c Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 20:01:57 +0800 Subject: [PATCH 086/132] Modify MySQLDQEOracle from TEXT to VARCHAR to prevent MEMORY-engine incompatibility --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index 8ddb6f315..bb55e585d 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -467,7 +467,7 @@ private List getErrors() throws SQLException { public void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { String tableName = table.getName(); - String addColumnRowID = String.format("ALTER TABLE %s ADD %s TEXT", tableName, COLUMN_ROWID); + String addColumnRowID = String.format("ALTER TABLE %s ADD %s VARCHAR(36)", tableName, COLUMN_ROWID); new SQLQueryAdapter(addColumnRowID).execute(state, false); state.getState().getLocalState().log(addColumnRowID); From f952b256ef9041b2c2b314fa809576625e2f9baa Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 18 Jun 2026 13:49:57 +0800 Subject: [PATCH 087/132] Fix MySQLDQEOracle false positives from non-deterministic ORDER BY LIMIT and known SELECT/DML error discrepancies Appends rowId as an ORDER BY tiebreaker to eliminate non-determinism when user columns contain duplicate values (e.g. NULLs) under LIMIT. Introduces isKnownSelectDMLDiscrepancy to suppress false positives from error codes that MySQL legitimately raises in UPDATE/DELETE but not SELECT (or vice versa) due to differing execution paths: WHERE-clause type coercion (1292, 1366), functional index maintenance (1030, 3751), and range optimizer memory limits (3170). --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index bb55e585d..cefcf4589 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -71,6 +71,10 @@ public String generateSelectStatement(AbstractTables tables, String tableN for (MySQLColumn column : Randomly.nonEmptySubset(mySQLTables.getColumns())) { orderColumns.add(column.getFullQualifiedName()); } + // rowId tiebreaker ensures ORDER BY LIMIT is deterministic when user columns have duplicate values + for (MySQLTable table : mySQLTables.getTables()) { + orderColumns.add(table.getName() + "." + COLUMN_ROWID); + } if (Randomly.getBooleanWithRatherLowProbability()) { generateLimit = true; @@ -185,7 +189,13 @@ public void check() throws SQLException { public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult updateResult) { if (updateResult.hasEmptyErrors()) { if (!selectResult.hasEmptyErrors()) { - return "SELECT has errors, but UPDATE does not."; + // Tolerate SELECT-only discrepancy errors (e.g. 1292 raised in SELECT but not UPDATE + // due to different short-circuit evaluation paths). + boolean selectHasNonDiscrepancyErrors = selectResult.getQueryErrors().stream() + .anyMatch(e -> !isKnownSelectDMLDiscrepancy(e)); + if (selectHasNonDiscrepancyErrors) { + return "SELECT has errors, but UPDATE does not."; + } } if (!selectResult.hasSameAccessedRows(updateResult)) { return "SELECT accessed different rows from UPDATE."; @@ -201,9 +211,14 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult } // update errors should all appear in the select errors + // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in UPDATE but not SELECT + // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); + if (isKnownSelectDMLDiscrepancy(updateError)) { + continue; + } if (!isFound(selectErrors, updateError)) { return "SELECT has different errors from UPDATE."; } @@ -247,7 +262,13 @@ private static boolean isFound(List selectErrors, SQLQueryError t public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { if (deleteResult.hasEmptyErrors()) { if (!selectResult.hasEmptyErrors()) { - return "SELECT has errors, but DELETE does not."; + // Tolerate SELECT-only discrepancy errors (e.g. 1292 raised in SELECT but not DELETE + // due to different short-circuit evaluation paths). + boolean selectHasNonDiscrepancyErrors = selectResult.getQueryErrors().stream() + .anyMatch(e -> !isKnownSelectDMLDiscrepancy(e)); + if (selectHasNonDiscrepancyErrors) { + return "SELECT has errors, but DELETE does not."; + } } if (!selectResult.hasSameAccessedRows(deleteResult)) { return "SELECT accessed different rows from DELETE."; @@ -263,9 +284,14 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult } // delete errors should all appear in the select errors + // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in DELETE but not SELECT + // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); + if (isKnownSelectDMLDiscrepancy(deleteError)) { + continue; + } if (!isFound(selectErrors, deleteError)) { return "SELECT has different errors from DELETE."; } @@ -349,6 +375,33 @@ private boolean hasDeleteSpecificErrors(SQLQueryResult deleteResult) { } + /* + * Errors that MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. These are + * acceptable discrepancies and should be skipped when checking that DML errors appear in SELECT errors. They are + * not treated as stop errors (hasStopErrors) so row comparison still proceeds normally. + * + * 1292: Truncated incorrect DOUBLE value — WHERE clause type coercion; MySQL may short-circuit in SELECT but + * evaluate fully in UPDATE/DELETE, raising this at ERROR level vs WARNING in SELECT. 1366: Incorrect + * integer/decimal/float value for column — same WHERE clause coercion discrepancy. 1030: Got error from storage + * engine — raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes so cannot + * produce this error. 3170: range_optimizer_max_mem_size exceeded — MySQL applies this memory budget differently + * for SELECT vs DML; the fallback full-scan still evaluates the WHERE predicate correctly. 3751: Data truncated for + * functional index — raised when a functional index expression truncates a value during DML; structurally + * impossible in SELECT. + */ + private static boolean isKnownSelectDMLDiscrepancy(SQLQueryError error) { + switch (error.getCode()) { + case 1030: + case 1292: + case 1366: + case 3170: + case 3751: + return true; + default: + return false; + } + } + private boolean hasStopErrors(SQLQueryResult queryResult) { return queryResult.getQueryErrors().stream() .anyMatch(error -> error.getLevel() == SQLQueryError.ErrorLevel.ERROR); From 2d50787b9f93539d35ccd41cf64aa35f2acdfe58 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 10:10:05 +0800 Subject: [PATCH 088/132] Move MySQLDQEOracle DML discrepancies into enum for maintainability --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index cefcf4589..429ef3c89 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -211,8 +211,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult } // update errors should all appear in the select errors - // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in UPDATE but not SELECT - // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. + // known SELECT/DML discrepancy errors are skipped: see KnownSelectDMLDiscrepancy for the full list. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); @@ -284,8 +283,7 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult } // delete errors should all appear in the select errors - // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in DELETE but not SELECT - // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. + // known SELECT/DML discrepancy errors are skipped: see KnownSelectDMLDiscrepancy for the full list. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); @@ -375,31 +373,36 @@ private boolean hasDeleteSpecificErrors(SQLQueryResult deleteResult) { } - /* - * Errors that MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. These are - * acceptable discrepancies and should be skipped when checking that DML errors appear in SELECT errors. They are - * not treated as stop errors (hasStopErrors) so row comparison still proceeds normally. - * - * 1292: Truncated incorrect DOUBLE value — WHERE clause type coercion; MySQL may short-circuit in SELECT but - * evaluate fully in UPDATE/DELETE, raising this at ERROR level vs WARNING in SELECT. 1366: Incorrect - * integer/decimal/float value for column — same WHERE clause coercion discrepancy. 1030: Got error from storage - * engine — raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes so cannot - * produce this error. 3170: range_optimizer_max_mem_size exceeded — MySQL applies this memory budget differently - * for SELECT vs DML; the fallback full-scan still evaluates the WHERE predicate correctly. 3751: Data truncated for - * functional index — raised when a functional index expression truncates a value during DML; structurally - * impossible in SELECT. - */ + // Errors MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. Acceptable + // discrepancies that should be skipped; not treated as stop errors so row comparison proceeds normally. + private enum KnownSelectDMLDiscrepancy { + // WHERE clause type coercion: MySQL may short-circuit in SELECT but evaluate fully in UPDATE/DELETE, + // raising this at ERROR level vs WARNING in SELECT. + TRUNCATED_DOUBLE_VALUE(1292), + // Same WHERE clause coercion discrepancy as TRUNCATED_DOUBLE_VALUE. + INCORRECT_COLUMN_VALUE(1366), + // Raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes. + STORAGE_ENGINE_ERROR(1030), + // MySQL applies this memory budget differently for SELECT vs DML; the fallback full-scan still + // evaluates the WHERE predicate correctly. + RANGE_OPTIMIZER_MEM_EXCEEDED(3170), + // Raised when a functional index expression truncates a value during DML; structurally impossible in SELECT. + FUNCTIONAL_INDEX_DATA_TRUNCATED(3751); + + private final int code; + + KnownSelectDMLDiscrepancy(int code) { + this.code = code; + } + } + private static boolean isKnownSelectDMLDiscrepancy(SQLQueryError error) { - switch (error.getCode()) { - case 1030: - case 1292: - case 1366: - case 3170: - case 3751: - return true; - default: - return false; + for (KnownSelectDMLDiscrepancy discrepancy : KnownSelectDMLDiscrepancy.values()) { + if (discrepancy.code == error.getCode()) { + return true; + } } + return false; } private boolean hasStopErrors(SQLQueryResult queryResult) { From 9efa585c35fe25594a0bf1f4157974e775925849 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 10:47:47 +0800 Subject: [PATCH 089/132] Ignore expected prefix-key-on-partitioned-table error in MySQLIndexGenerator MySQL rejects CREATE INDEX with a prefix key part (e.g. c0(3)) on a column that participates in PARTITION BY KEY(). MySQLIndexGenerator generated these without checking partition membership and did not include this error in ExpectedErrors, causing checkException to escalate it to a fatal AssertionError. Adds the error substring to the expected errors list. --- src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index 550893db5..028886831 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -120,6 +120,8 @@ public SQLQueryAdapter create() { errors.add("Data truncated for functional index"); errors.add("used in key specification without a key length"); errors.add("Row size too large"); // seems to happen together with MIN_ROWS in the table declaration + errors.add("in the PARTITION BY KEY() clause is not supported"); // prefix key parts disallowed on + // KEY-partitioned columns return new SQLQueryAdapter(string, errors, true); } From 8fd48d3d3d092485a0c730887a083f925cc4c2ca Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 13:05:14 +0800 Subject: [PATCH 090/132] Fix YugabyteDB CI by adding ysql_yb_enable_listen_notify flag so that LISTEN and NOTIFY statements are enabled --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c511fc0a..91d19197f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -646,7 +646,7 @@ jobs: - name: Set up Yugabyte run: | docker pull yugabytedb/yugabyte:latest - docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false + docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false --tserver_flags="ysql_yb_enable_listen_notify=true" --master_flags="ysql_yb_enable_listen_notify=true" until pg_isready -h localhost -p 5433; do sleep 1; done - name: Run Tests run: | From 376b9d4c1f6b3118a4926af35e35d163ce73beed Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 13:34:22 +0800 Subject: [PATCH 091/132] Add YCQL readiness check for YugabyteDB so that TestYCQL does not run prematurely --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91d19197f..02507d58f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -648,6 +648,7 @@ jobs: docker pull yugabytedb/yugabyte:latest docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false --tserver_flags="ysql_yb_enable_listen_notify=true" --master_flags="ysql_yb_enable_listen_notify=true" until pg_isready -h localhost -p 5433; do sleep 1; done + until nc -z localhost 9042; do sleep 1; done - name: Run Tests run: | YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLNoREC test From 2821cbd02890185a210739c42028356b4aa41222 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 11 Jul 2026 20:01:22 +0800 Subject: [PATCH 092/132] Remove NoRECBase and DocumentRemovalOracleBase, as these were dead code --- .../oracle/DocumentRemovalOracleBase.java | 29 ------------------- src/sqlancer/common/oracle/NoRECBase.java | 26 ----------------- 2 files changed, 55 deletions(-) delete mode 100644 src/sqlancer/common/oracle/DocumentRemovalOracleBase.java delete mode 100644 src/sqlancer/common/oracle/NoRECBase.java diff --git a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java b/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java deleted file mode 100644 index b6c0ee509..000000000 --- a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java +++ /dev/null @@ -1,29 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.GlobalState; -import sqlancer.common.gen.ExpressionGenerator; - -public abstract class DocumentRemovalOracleBase> implements TestOracle { - - protected E predicate; - - protected final S state; - - protected DocumentRemovalOracleBase(S state) { - this.state = state; - } - - protected void initializeDocumentRemovalOracle() { - ExpressionGenerator gen = getGen(); - if (gen == null) { - throw new IllegalStateException(); - } - predicate = gen.generatePredicate(); - if (predicate == null) { - throw new IllegalStateException(); - } - } - - protected abstract ExpressionGenerator getGen(); - -} diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java deleted file mode 100644 index 2ac0dbb43..000000000 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ /dev/null @@ -1,26 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; -import sqlancer.common.query.ExpectedErrors; - -public abstract class NoRECBase> implements TestOracle { - - protected final S state; - protected final ExpectedErrors errors = new ExpectedErrors(); - protected final StateLogger logger; - protected final MainOptions options; - protected final SQLConnection con; - protected String optimizedQueryString; - protected String unoptimizedQueryString; - - protected NoRECBase(S state) { - this.state = state; - this.con = state.getConnection(); - this.logger = state.getLogger(); - this.options = state.getOptions(); - } - -} From f984df4cece25768c588774e992f36681bd48655 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 13 Jul 2026 18:30:37 +0800 Subject: [PATCH 093/132] Fix TLPWhereReproducer bug that prevented reducer from performing any reduction --- src/sqlancer/MainOptions.java | 4 ++-- src/sqlancer/common/oracle/TLPWhereOracle.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index a5142fcf0..25b769312 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -126,10 +126,10 @@ public class MainOptions { @Parameter(names = "--serialize-reproduce-state", description = "Serialize the state to reproduce") private boolean serializeReproduceState = false; // NOPMD - @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") + @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer. Implemented for TLP WHERE and NoREC only") private boolean useReducer = false; // NOPMD - @Parameter(names = "--reduce-ast", description = "EXPERIMENTAL perform AST reduction after statement reduction") + @Parameter(names = "--reduce-ast", description = "EXPERIMENTAL Perform AST reduction after statement reduction") private boolean reduceAST = false; // NOPMD @Parameter(names = "--statement-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the statement reducer will do") diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 14834a62f..31ecc4481 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -106,15 +106,15 @@ public void check() throws SQLException { select.setWhereClause(predicates.isNullPredicate); String thirdQueryString = select.asString(); + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, + firstResultSet, orderBy); + List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); - - reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - firstResultSet, orderBy); } @Override From 73157498bda4fd958e8ac829f39cb4560b2e1284 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 13 Jul 2026 19:06:42 +0800 Subject: [PATCH 094/132] Fix bug where reducer logs were being overwritten, causing logs to end up empty --- src/sqlancer/Main.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index f273f5b95..33cf5fbe3 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -502,11 +502,13 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } - try { - logger.getReduceFileWriter().close(); - logger.reduceFileWriter = null; - } catch (IOException e) { - throw new AssertionError(e); + if (logger.reduceFileWriter != null) { + try { + logger.reduceFileWriter.close(); + logger.reduceFileWriter = null; + } catch (IOException e) { + throw new AssertionError(e); + } } throw new AssertionError("Found a potential bug, please check reducer log for detail."); From 2b3b28b051e75dceb919daf12638b0a0dc817ffe Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 14 Jul 2026 13:46:43 +0800 Subject: [PATCH 095/132] Fix bug where reducer always reduced to single statement --- src/sqlancer/common/oracle/NoRECOracle.java | 16 ++++++++-- .../common/oracle/TLPWhereOracle.java | 29 ++++++++++++------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index caf3dff87..0f043a07f 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -1,7 +1,6 @@ package sqlancer.common.oracle; import java.sql.SQLException; -import java.util.Objects; import java.util.function.Function; import sqlancer.IgnoreMeException; @@ -42,7 +41,20 @@ private static class NoRECReproducer> implements @Override public boolean bugStillTriggers(G globalState) { - return !Objects.equals(optimizedQuery.apply(globalState), unoptimizedQuery.apply(globalState)); + int optimizedCount; + int unoptimizedCount; + try { + optimizedCount = optimizedQuery.apply(globalState); + unoptimizedCount = unoptimizedQuery.apply(globalState); + } catch (RuntimeException | AssertionError e) { + // the queries could not be executed on the reduced database (e.g., a statement they + // depend on was removed), which is not the count mismatch that is being reduced + return false; + } + if (optimizedCount == -1 || unoptimizedCount == -1) { + return false; + } + return optimizedCount != unoptimizedCount; } } diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 31ecc4481..60b8d0c29 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -34,30 +34,37 @@ private class TLPWhereReproducer implements Reproducer { final String secondQueryString; final String thirdQueryString; final String originalQueryString; - final List resultSet; final boolean orderBy; TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, - String originalQueryString, List resultSet, boolean orderBy) { + String originalQueryString, boolean orderBy) { this.firstQueryString = firstQueryString; this.secondQueryString = secondQueryString; this.thirdQueryString = thirdQueryString; this.originalQueryString = originalQueryString; - this.resultSet = resultSet; this.orderBy = orderBy; } @Override public boolean bugStillTriggers(G globalState) { + List firstResultSet; + List combinedString = new ArrayList<>(); + List secondResultSet; try { - List combinedString1 = new ArrayList<>(); - List secondResultSet1 = ComparatorHelper.getCombinedResultSet(firstQueryString, - secondQueryString, thirdQueryString, combinedString1, !orderBy, globalState, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet1, originalQueryString, - combinedString1, globalState); - } catch (AssertionError triggeredError) { + firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + globalState); + secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, globalState, errors); + } catch (SQLException | RuntimeException | AssertionError e) { + // the queries could not be executed on the reduced database (e.g., a statement they + // depend on was removed), which is not the result set mismatch that is being reduced + return false; + } + try { + ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, + combinedString, globalState); + } catch (AssertionError resultSetMismatch) { return true; - } catch (SQLException ignored) { } return false; } @@ -107,7 +114,7 @@ public void check() throws SQLException { String thirdQueryString = select.asString(); reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - firstResultSet, orderBy); + orderBy); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, From 7d11ca0bb1b76446c18751cd46058b2c9c801ce8 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 14 Jul 2026 13:56:38 +0800 Subject: [PATCH 096/132] Extend reducer to allow reducing DBMS errors (i.e. errors other than oracle logic bugs) --- src/sqlancer/Main.java | 14 +++- src/sqlancer/Reproducer.java | 10 +++ src/sqlancer/common/oracle/NoRECOracle.java | 67 ++++++++++++---- .../common/oracle/TLPWhereOracle.java | 76 ++++++++++++++++--- .../common/oracle/TestOracleUtils.java | 23 ++++++ 5 files changed, 162 insertions(+), 28 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 33cf5fbe3..d65649049 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -502,10 +502,18 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } - if (logger.reduceFileWriter != null) { + String bugInformation = reproducer.getBugInformation(); + if (bugInformation != null) { + // log through newGlobalState's logger: it already holds the reduce file + // writer, and opening it through another StateLogger truncates the file + newGlobalState.getLogger().logReducer(bugInformation); + } + + StateLogger reduceLogger = newGlobalState.getLogger(); + if (reduceLogger.reduceFileWriter != null) { try { - logger.reduceFileWriter.close(); - logger.reduceFileWriter = null; + reduceLogger.reduceFileWriter.close(); + reduceLogger.reduceFileWriter = null; } catch (IOException e) { throw new AssertionError(e); } diff --git a/src/sqlancer/Reproducer.java b/src/sqlancer/Reproducer.java index ef64bd0fe..460cc810d 100644 --- a/src/sqlancer/Reproducer.java +++ b/src/sqlancer/Reproducer.java @@ -2,4 +2,14 @@ public interface Reproducer> { boolean bugStillTriggers(G globalState); + + /** + * Describes how to trigger the bug on the database set up by the reduced statements (e.g., the oracle queries to + * run and the failure to expect), so that the reduced test case is complete without the reproducer object. + * + * @return a human-readable description, or null if the reproducer does not provide one + */ + default String getBugInformation() { + return null; + } } diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 0f043a07f..35f0c26af 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -33,10 +33,19 @@ public class NoRECOracle, J extends Join, private static class NoRECReproducer> implements Reproducer { private final Function optimizedQuery; private final Function unoptimizedQuery; - - NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery) { + private final String optimizedQueryString; + private final String unoptimizedQueryString; + // null if the original bug is a count mismatch; otherwise, the message of the unexpected + // DBMS error that the original queries triggered + private final String expectedErrorMessage; + + NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, + String optimizedQueryString, String unoptimizedQueryString, String expectedErrorMessage) { this.optimizedQuery = optimizedQuery; this.unoptimizedQuery = unoptimizedQuery; + this.optimizedQueryString = optimizedQueryString; + this.unoptimizedQueryString = unoptimizedQueryString; + this.expectedErrorMessage = expectedErrorMessage; } @Override @@ -46,9 +55,16 @@ public boolean bugStillTriggers(G globalState) { try { optimizedCount = optimizedQuery.apply(globalState); unoptimizedCount = unoptimizedQuery.apply(globalState); - } catch (RuntimeException | AssertionError e) { - // the queries could not be executed on the reduced database (e.g., a statement they - // depend on was removed), which is not the count mismatch that is being reduced + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs return false; } if (optimizedCount == -1 || unoptimizedCount == -1) { @@ -56,6 +72,22 @@ public boolean bugStillTriggers(G globalState) { } return optimizedCount != unoptimizedCount; } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the row counts of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); + sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + return sb.toString(); + } } public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { @@ -94,21 +126,28 @@ public void check() throws SQLException { state.getLogger().writeCurrent(unoptimizedQueryString); } - int optimizedCount = shouldUseAggregate ? extractCounts(optimizedQueryString, errors, state) - : countRows(optimizedQueryString, errors, state); - int unoptimizedCount = extractCounts(unoptimizedQueryString, errors, state); + Function optimizedQuery = state -> shouldUseAggregate + ? extractCounts(optimizedQueryString, errors, state) : countRows(optimizedQueryString, errors, state); + Function unoptimizedQuery = state -> extractCounts(unoptimizedQueryString, errors, state); + + int optimizedCount; + int unoptimizedCount; + try { + optimizedCount = optimizedQuery.apply(state); + unoptimizedCount = unoptimizedQuery.apply(state); + } catch (AssertionError unexpectedError) { + reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, + unoptimizedQueryString, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } if (optimizedCount == -1 || unoptimizedCount == -1) { throw new IgnoreMeException(); } if (unoptimizedCount != optimizedCount) { - Function optimizedQuery = state -> shouldUseAggregate - ? extractCounts(optimizedQueryString, errors, state) - : countRows(optimizedQueryString, errors, state); - - Function unoptimizedQuery = state -> extractCounts(unoptimizedQueryString, errors, state); - reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery); + reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, + unoptimizedQueryString, null); String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 60b8d0c29..968064158 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -35,14 +35,18 @@ private class TLPWhereReproducer implements Reproducer { final String thirdQueryString; final String originalQueryString; final boolean orderBy; + // null if the original bug is a result set mismatch; otherwise, the message of the + // unexpected DBMS error that the original queries triggered + final String expectedErrorMessage; TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, - String originalQueryString, boolean orderBy) { + String originalQueryString, boolean orderBy, String expectedErrorMessage) { this.firstQueryString = firstQueryString; this.secondQueryString = secondQueryString; this.thirdQueryString = thirdQueryString; this.originalQueryString = originalQueryString; this.orderBy = orderBy; + this.expectedErrorMessage = expectedErrorMessage; } @Override @@ -53,11 +57,23 @@ public boolean bugStillTriggers(G globalState) { try { firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + if (firstQueryString == null) { + // the original bug was a DBMS error on the original query alone, which no + // longer occurs + return false; + } secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, globalState, errors); - } catch (SQLException | RuntimeException | AssertionError e) { - // the queries could not be executed on the reduced database (e.g., a statement they - // depend on was removed), which is not the result set mismatch that is being reduced + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs return false; } try { @@ -68,6 +84,32 @@ public boolean bugStillTriggers(G globalState) { } return false; } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (firstQueryString != null) { + if (orderBy) { + sb.append("-- ").append(firstQueryString).append(';').append(System.lineSeparator()); + sb.append("-- ").append(secondQueryString).append(';').append(System.lineSeparator()); + sb.append("-- ").append(thirdQueryString).append(';').append(System.lineSeparator()); + } else { + sb.append("-- ").append(firstQueryString).append(" UNION ALL ").append(secondQueryString) + .append(" UNION ALL ").append(thirdQueryString).append(';') + .append(System.lineSeparator()); + } + } + return sb.toString(); + } } public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { @@ -96,8 +138,14 @@ public void check() throws SQLException { String originalQueryString = select.asString(); generatedQueryString = originalQueryString; - List firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - state); + List firstResultSet; + try { + firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + } catch (AssertionError unexpectedError) { + reproducer = new TLPWhereReproducer(null, null, null, originalQueryString, false, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } boolean orderBy = Randomly.getBooleanWithSmallProbability(); if (orderBy) { @@ -113,13 +161,19 @@ public void check() throws SQLException { select.setWhereClause(predicates.isNullPredicate); String thirdQueryString = select.asString(); - reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - orderBy); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); + List secondResultSet; + try { + secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + } catch (AssertionError unexpectedError) { + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, + originalQueryString, orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, + orderBy, null); ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); } diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java index bab2e26c9..5233cfa80 100644 --- a/src/sqlancer/common/oracle/TestOracleUtils.java +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -34,6 +34,29 @@ public static final class PredicateVariants, C extends A return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables())); } + /** + * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the + * AssertionError that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it + * to check that a reduced test case still triggers the same error, rather than an unrelated one introduced by the + * reduction itself. + * + * @param error + * the AssertionError wrapping the DBMS error + * + * @return the message of the innermost cause that has one, or the error's own message + */ + public static String getUnexpectedErrorMessage(AssertionError error) { + String message = error.getMessage(); + Throwable current = error.getCause(); + while (current != null) { + if (current.getMessage() != null) { + message = current.getMessage(); + } + current = current.getCause(); + } + return message; + } + public static , T extends AbstractTable, C extends AbstractTableColumn> PredicateVariants initializeTernaryPredicateVariants( PartitionGenerator gen, E predicate) { if (gen == null) { From a02b149a65598b8af893717ceacee764f9d6c395 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 14 Jul 2026 14:47:01 +0800 Subject: [PATCH 097/132] Clarify logs when --use-reducer is applied (the log outside of the reduce directory gives the final reduced test case, while the log inside the reduce directory gives the step-by-step reduction) --- src/sqlancer/ASTBasedReducer.java | 6 +- src/sqlancer/Main.java | 76 +++++++++++++------ src/sqlancer/StateToReproduce.java | 4 + src/sqlancer/StatementReducer.java | 3 +- .../common/oracle/TLPWhereOracle.java | 3 +- .../common/oracle/TestOracleUtils.java | 7 +- 6 files changed, 66 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java index 876a2da12..f9468af76 100644 --- a/src/sqlancer/ASTBasedReducer.java +++ b/src/sqlancer/ASTBasedReducer.java @@ -103,8 +103,7 @@ public void reduce(G state, Reproducer reproducer, G newGlobalState) throws E }); if (!initFlag) { - newGlobalState.getLogger() - .logReducer("warning: failed parsing the statement at transformer : " + t); + System.out.println("Error when parsing the statement at transformer :" + t); continue; } t.apply(); @@ -114,7 +113,8 @@ public void reduce(G state, Reproducer reproducer, G newGlobalState) throws E } while (observeChange); newGlobalState.getState().setStatements(new ArrayList<>(reducedStatements)); - newGlobalState.getLogger().logReduced(newGlobalState.getState()); + newGlobalState.getLogger().logReduced(newGlobalState.getState(), + "AST-based reduction finished; the following statements remain"); } public boolean bugStillTriggers() throws Exception { diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index d65649049..47ba2aedf 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -80,6 +80,9 @@ public static final class StateLogger { private FileWriter queryPlanFileWriter; private FileWriter reduceFileWriter; private Path reproduceFilePath; + private List> reduceSetupStatements; + private String reduceBugInformation; + private int nrReductionAttempts; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; @@ -262,33 +265,32 @@ public void writeQueryPlan(String queryPlan) { } } - public void logReducer(String reducerLog) { - FileWriter reduceFileWriter = getReduceFileWriter(); - - StringBuilder sb = new StringBuilder(); - sb.append("[reducer log] "); - sb.append(reducerLog); - try { - reduceFileWriter.write(sb.toString()); - } catch (IOException e) { - throw new AssertionError(e); - } finally { - try { - reduceFileWriter.flush(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } + public void setReductionContext(List> setupStatements, String bugInformation) { + this.reduceSetupStatements = setupStatements; + this.reduceBugInformation = bugInformation; } public void logReduced(StateToReproduce state) { + nrReductionAttempts++; + logReduced(state, "Reduction attempt " + nrReductionAttempts + + ": the bug was still triggered with the following statements"); + } + + public void logReduced(StateToReproduce state, String description) { FileWriter reduceFileWriter = getReduceFileWriter(); StringBuilder sb = new StringBuilder(); - for (Query s : state.getStatements()) { - sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); + sb.append("-- ").append(description).append(System.lineSeparator()); + if (reduceSetupStatements != null && !reduceSetupStatements.isEmpty()) { + appendStatements(sb, reduceSetupStatements); + // e.g. DROP DATABASE IF EXISTS db; CREATE DATABASE db; USE db; + // these statements are executed at the start of every test case and are never reduced + } + appendStatements(sb, state.getStatements()); + if (reduceBugInformation != null) { + sb.append(reduceBugInformation); } + sb.append(System.lineSeparator()); try { reduceFileWriter.write(sb.toString()); @@ -305,6 +307,12 @@ public void logReduced(StateToReproduce state) { } + private void appendStatements(StringBuilder sb, List> statements) { + for (Query s : statements) { + sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); + } + } + public void logException(Throwable reduce, StateToReproduce state) { Loggable stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); @@ -460,6 +468,9 @@ public void run() throws Exception { if (options.logEachSelect()) { logger.writeCurrent(state.getState()); } + // statements logged so far stem from the database setup (e.g., DROP DATABASE IF + // EXISTS, CREATE DATABASE, USE), performed by createDatabase + int nrSetupStatements = stateToRepro.getStatements().size(); Reproducer reproducer = null; if (options.enableQPG()) { provider.generateAndTestDatabaseWithQueryPlanGuidance(state); @@ -484,6 +495,17 @@ public void run() throws Exception { logger.getReduceFileWriter().write("current oracle does not support experimental reducer."); throw new IgnoreMeException(); } + + // reduce only the generation statements: the database setup (logged by + // createDatabase) is re-executed by the reducers for every candidate, and the + // oracle queries (logged by the oracle's local state) by the reproducer + List> allStatements = new ArrayList<>(stateToRepro.getStatements()); + List> setupStatements = new ArrayList<>(allStatements.subList(0, nrSetupStatements)); + List> oracleQueryStatements = stateToRepro.getLocalState() == null ? new ArrayList<>() + : new ArrayList<>(stateToRepro.getLocalState().getStatements()); + stateToRepro.setStatements(new ArrayList<>(allStatements.subList(nrSetupStatements, + allStatements.size() - oracleQueryStatements.size()))); + G newGlobalState = createGlobalState(); newGlobalState.setState(stateToRepro); newGlobalState.setRandomly(r); @@ -493,6 +515,7 @@ public void run() throws Exception { QueryManager newManager = new QueryManager<>(newGlobalState); newGlobalState.setStateLogger(new StateLogger(databaseName, provider, options)); newGlobalState.setManager(newManager); + newGlobalState.getLogger().setReductionContext(setupStatements, reproducer.getBugInformation()); Reducer reducer = new StatementReducer<>(provider); reducer.reduce(state, reproducer, newGlobalState); @@ -502,11 +525,18 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } + // reassemble the statements so that the main log looks like one produced + // without the reducer, with the generation statements replaced by the reduced + // ones and the oracle queries at the end + List> finalStatements = new ArrayList<>(setupStatements); + finalStatements.addAll(stateToRepro.getStatements()); + finalStatements.addAll(oracleQueryStatements); + stateToRepro.setStatements(finalStatements); String bugInformation = reproducer.getBugInformation(); if (bugInformation != null) { - // log through newGlobalState's logger: it already holds the reduce file - // writer, and opening it through another StateLogger truncates the file - newGlobalState.getLogger().logReducer(bugInformation); + for (String line : bugInformation.split(System.lineSeparator())) { + stateToRepro.logStatement(line); + } } StateLogger reduceLogger = newGlobalState.getLogger(); diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index e44d0ccf6..17bb367fd 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -128,6 +128,10 @@ public void log(String s) { statements.add(databaseProvider.getLoggableFactory().getQueryForStateToReproduce(s)); } + public List> getStatements() { + return Collections.unmodifiableList(statements); + } + @Override public void close() { if (!success) { diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java index e066aca84..6545fb2af 100644 --- a/src/sqlancer/StatementReducer.java +++ b/src/sqlancer/StatementReducer.java @@ -77,7 +77,8 @@ && hasNotReachedLimit(currentReduceTime, maxReduceTime)) { // System.out.println("Reduced query:"); // printQueries(knownToReproduceBugStatements); newGlobalState.getState().setStatements(new ArrayList<>(knownToReproduceBugStatements)); - newGlobalState.getLogger().logReduced(newGlobalState.getState()); + newGlobalState.getLogger().logReduced(newGlobalState.getState(), + "Statement reduction finished; the following statements remain"); } diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 968064158..8fc87ac3d 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -104,8 +104,7 @@ public String getBugInformation() { sb.append("-- ").append(thirdQueryString).append(';').append(System.lineSeparator()); } else { sb.append("-- ").append(firstQueryString).append(" UNION ALL ").append(secondQueryString) - .append(" UNION ALL ").append(thirdQueryString).append(';') - .append(System.lineSeparator()); + .append(" UNION ALL ").append(thirdQueryString).append(';').append(System.lineSeparator()); } } return sb.toString(); diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java index 5233cfa80..9bef86762 100644 --- a/src/sqlancer/common/oracle/TestOracleUtils.java +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -35,10 +35,9 @@ public static final class PredicateVariants, C extends A } /** - * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the - * AssertionError that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it - * to check that a reduced test case still triggers the same error, rather than an unrelated one introduced by the - * reduction itself. + * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the AssertionError + * that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it to check that a + * reduced test case still triggers the same error, rather than an unrelated one introduced by the reduction itself. * * @param error * the AssertionError wrapping the DBMS error From abbdeed6cbe4bad3c9c7e43b92c880ab47cedf5a Mon Sep 17 00:00:00 2001 From: splf Date: Mon, 27 Jul 2026 23:03:27 +0500 Subject: [PATCH 098/132] Fix MySQL index hint syntax in the DQP hint generator MySQL's grammar for index-level optimizer hints is hint_name(tbl_name index_name [, index_name] ...): a space separates the table name from the first index name, not a comma. MySQLHintGenerator emitted NO_INDEX(t0, PRIMARY), which MySQL rejects with warning 1064 and silently ignores, leaving the query plan unchanged. This affects the 16 of 32 hint kinds that go through indexesHint(). --- src/sqlancer/mysql/gen/MySQLHintGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLHintGenerator.java b/src/sqlancer/mysql/gen/MySQLHintGenerator.java index 141aea279..dc6138b23 100644 --- a/src/sqlancer/mysql/gen/MySQLHintGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLHintGenerator.java @@ -170,7 +170,7 @@ private void indexesHint(String string) { MySQLTable table = Randomly.fromList(tables); List allIndexes = table.getIndexes(); sb.append(table.getName()); - sb.append(", "); + sb.append(" "); if (allIndexes.isEmpty()) { sb.append("PRIMARY"); } else { From eb5e013a227eae7834c5b1f3865c451e57bb3f51 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 9 Jul 2026 13:21:12 +0800 Subject: [PATCH 099/132] Add EET (Equivalent Expression Transformation) oracle for MySQL This commit implements EET from Jiang & Su, "Detecting Logic Bugs in Database Engines via Equivalent Expression Transformation" (OSDI'24). Common, DBMS-independent core: - EETTransformation: the 7 transformation rules of the paper's Table 2. - EETNodeFactory: primitive node constructors the rules are built from. - EETGenerator: generator interface an EET-capable DBMS implements. - EETOracle: orchestration and result comparison (paper Figure 4), with a Reproducer for confirmation/reduction. MySQL-specific implementation: - MySQLEETTransformer recursively rewrites the MySQL AST, rebuilding each node from its transformed children and threading a boolean/scalar context flag so the determined-boolean rules are only applied where sound. - MySQLEETNodeFactory builds the MySQL nodes; MySQLExpressionGenerator implements EETGenerator; MySQLUnaryPrefixOperation exposes getOp(); MySQLOracleFactory registers the EET oracle. - MySQLTableGenerator skips ZEROFILL for EET (as for TLP_WHERE/PQS/DQP) to avoid display-metadata-only false positives while preserving search space. Runs via: java -jar target/sqlancer-*.jar mysql --oracle EET --- src/sqlancer/common/gen/EETGenerator.java | 44 +++++++ .../common/oracle/EETNodeFactory.java | 39 ++++++ src/sqlancer/common/oracle/EETOracle.java | 121 +++++++++++++++++ .../common/oracle/EETTransformation.java | 83 ++++++++++++ src/sqlancer/mysql/MySQLOracleFactory.java | 10 ++ .../mysql/ast/MySQLUnaryPrefixOperation.java | 4 + .../mysql/gen/MySQLEETNodeFactory.java | 70 ++++++++++ .../mysql/gen/MySQLEETTransformer.java | 124 ++++++++++++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 13 +- .../mysql/gen/MySQLTableGenerator.java | 2 +- 10 files changed, 508 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/common/gen/EETGenerator.java create mode 100644 src/sqlancer/common/oracle/EETNodeFactory.java create mode 100644 src/sqlancer/common/oracle/EETOracle.java create mode 100644 src/sqlancer/common/oracle/EETTransformation.java create mode 100644 src/sqlancer/mysql/gen/MySQLEETNodeFactory.java create mode 100644 src/sqlancer/mysql/gen/MySQLEETTransformer.java diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java new file mode 100644 index 000000000..f468ff6d2 --- /dev/null +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -0,0 +1,44 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like + * the other oracle generators), an EET generator can transform an expression into a semantically equivalent one + * according to the EET transformation rules. + */ +public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + EETGenerator setTablesAndColumns(AbstractTables tables); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + E generateBooleanExpression(); + + /** + * Transforms an expression into a semantically equivalent one (the core of EET). Typically this recursively + * traverses the expression's AST and replaces sub-expressions with equivalent ones. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (e.g. a WHERE predicate); this controls + * which transformation rules are applicable + * + * @return a semantically equivalent expression + */ + E transformExpression(E expr, boolean booleanContext); +} diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java new file mode 100644 index 000000000..2cc933ced --- /dev/null +++ b/src/sqlancer/common/oracle/EETNodeFactory.java @@ -0,0 +1,39 @@ +package sqlancer.common.oracle; + +/** + * Factory for constructing the AST nodes needed by the {@link EETTransformation equivalent expression transformation} + * rules. Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface + * lets the (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. + * + * @param + * the DBMS-specific expression type + */ +public interface EETNodeFactory { + + /** Builds {@code left AND right}. */ + E and(E left, E right); + + /** Builds {@code left OR right}. */ + E or(E left, E right); + + /** Builds {@code NOT expr}. */ + E not(E expr); + + /** Builds {@code expr IS NULL}. */ + E isNull(E expr); + + /** Builds {@code expr IS NOT NULL}. */ + E isNotNull(E expr); + + /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ + E caseWhen(E condition, E thenExpr, E elseExpr); + + /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ + E generateBooleanExpression(); + + /** + * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not + * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + */ + boolean isCaseWhenApplicable(E expr); +} diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java new file mode 100644 index 000000000..0f192f9ab --- /dev/null +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -0,0 +1,121 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.EETGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * EET (Equivalent Expression Transformation) oracle, based on "Detecting Logic Bugs in Database Engines via Equivalent + * Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * The oracle generates a random query and then transforms its expressions (the WHERE predicate and the fetch columns) + * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation preserves + * semantics, the original and the transformed query must return the same result set; any discrepancy indicates a logic + * bug in the DBMS. + */ +public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private EETGenerator gen; + private final ExpectedErrors errors; + + private Reproducer reproducer; + private String generatedQueryString; + + private final class EETReproducer implements Reproducer { + private final String originalQueryString; + private final String transformedQueryString; + private final List resultSet; + + EETReproducer(String originalQueryString, String transformedQueryString, List resultSet) { + this.originalQueryString = originalQueryString; + this.transformedQueryString = transformedQueryString; + this.resultSet = resultSet; + } + + @Override + public boolean bugStillTriggers(G globalState) { + try { + List transformedResultSet = ComparatorHelper + .getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, transformedResultSet, originalQueryString, + List.of(transformedQueryString), globalState); + } catch (AssertionError triggeredError) { + return true; + } catch (SQLException ignored) { + } + return false; + } + } + + public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + reproducer = null; + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + Z select = gen.generateSelect(); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + List fetchColumns = gen.generateFetchColumns(true); + select.setFetchColumns(fetchColumns); + E whereClause = gen.generateBooleanExpression(); + select.setWhereClause(whereClause); + + String originalQueryString = select.asString(); + generatedQueryString = originalQueryString; + List originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, + // while the WHERE clause is evaluated in a boolean context. + List transformedFetchColumns = fetchColumns.stream().map(c -> gen.transformExpression(c, false)) + .collect(Collectors.toList()); + select.setFetchColumns(transformedFetchColumns); + select.setWhereClause(gen.transformExpression(whereClause, true)); + + String transformedQueryString = select.asString(); + List transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, + errors, state); + + ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, + List.of(transformedQueryString), state); + + reproducer = new EETReproducer(originalQueryString, transformedQueryString, originalResultSet); + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/common/oracle/EETTransformation.java b/src/sqlancer/common/oracle/EETTransformation.java new file mode 100644 index 000000000..ab9b7fefe --- /dev/null +++ b/src/sqlancer/common/oracle/EETTransformation.java @@ -0,0 +1,83 @@ +package sqlancer.common.oracle; + +import sqlancer.Randomly; + +/** + * Implements the semantic-preserving expression transformation rules of EET (Equivalent Expression Transformation, Jiang + * & Su, OSDI'24), Table 2. Given an expression, {@link #applyRandomRule} returns a semantically equivalent + * expression built from the primitives provided by an {@link EETNodeFactory}. The rules are DBMS-independent; only the + * node construction (via the factory) is DBMS-specific. + * + *

+ * The rules rely on two always-determined boolean expressions built from an arbitrary boolean {@code p}: + *

    + *
  • {@code true_expr(p) = p OR (NOT p) OR (p IS NULL)}, which always evaluates to TRUE, and
    • + *
  • {@code false_expr(p) = p AND (NOT p) AND (p IS NOT NULL)}, which always evaluates to FALSE.
    • + *
+ * + * @param + * the DBMS-specific expression type + */ +public class EETTransformation { + + private final EETNodeFactory factory; + + public EETTransformation(EETNodeFactory factory) { + this.factory = factory; + } + + // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE + private E trueExpr() { + E p = factory.generateBooleanExpression(); + return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); + } + + // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE + private E falseExpr() { + E p = factory.generateBooleanExpression(); + return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); + } + + /** + * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen, applicable + * transformation rule. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (e.g. a WHERE predicate or an operand of a + * logical operator). Only in a boolean context may the determined-boolean rules (No. 1 and 2), which + * reduce the expression to a boolean value, be applied; in a scalar context they would change the + * expression's value/type and are therefore excluded. + * + * @return a semantically equivalent expression + */ + public E applyRandomRule(E expr, boolean booleanContext) { + int rule; + if (booleanContext) { + // Rules No. 1-6 are all value-preserving in a boolean context. + rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); + } else { + if (!factory.isCaseWhenApplicable(expr)) { + return expr; // rule No. 7: transform the expression to itself + } + // In a scalar context only the CASE WHEN rules preserve the exact value and type. + rule = Randomly.fromOptions(3, 4, 5, 6); + } + switch (rule) { + case 1: // bool_expr => false_expr OR bool_expr + return factory.or(falseExpr(), expr); + case 2: // bool_expr => true_expr AND bool_expr + return factory.and(trueExpr(), expr); + case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END + return factory.caseWhen(falseExpr(), expr, expr); + case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END + return factory.caseWhen(trueExpr(), expr, expr); + case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END + case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END + return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); + default: + throw new AssertionError(rule); + } + } +} diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index 83e08677a..ed5ddc489 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -5,6 +5,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.EETOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; @@ -82,5 +83,14 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLDQEOracle(globalState); } + }, + EET { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + return new EETOracle<>(globalState, gen, expectedErrors); + } }; } diff --git a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java index c87048a4b..45a1fe888 100644 --- a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java +++ b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java @@ -62,6 +62,10 @@ public MySQLUnaryPrefixOperation(MySQLExpression expr, MySQLUnaryPrefixOperator super(expr, op); } + public MySQLUnaryPrefixOperator getOp() { + return op; + } + @Override public MySQLConstant getExpectedValue() { MySQLConstant subExprVal = expr.getExpectedValue(); diff --git a/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java b/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java new file mode 100644 index 000000000..33f3ab587 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java @@ -0,0 +1,70 @@ +package sqlancer.mysql.gen; + +import java.util.List; + +import sqlancer.common.oracle.EETNodeFactory; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; + +/** + * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformation EET transformation} + * rules. + */ +public class MySQLEETNodeFactory implements EETNodeFactory { + + private final MySQLExpressionGenerator gen; + + public MySQLEETNodeFactory(MySQLExpressionGenerator gen) { + this.gen = gen; + } + + @Override + public MySQLExpression and(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); + } + + @Override + public MySQLExpression or(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); + } + + @Override + public MySQLExpression not(MySQLExpression expr) { + return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); + } + + @Override + public MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); + } + + @Override + public MySQLExpression isNotNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); + } + + @Override + public MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { + return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); + } + + @Override + public MySQLExpression generateBooleanExpression() { + return gen.generateBooleanExpression(); + } + + @Override + public boolean isCaseWhenApplicable(MySQLExpression expr) { + // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET + // paper); aggregates are excluded to avoid placing them in invalid contexts. + return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); + } +} diff --git a/src/sqlancer/mysql/gen/MySQLEETTransformer.java b/src/sqlancer/mysql/gen/MySQLEETTransformer.java new file mode 100644 index 000000000..000e29bb6 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLEETTransformer.java @@ -0,0 +1,124 @@ +package sqlancer.mysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.oracle.EETTransformation; +import sqlancer.mysql.ast.MySQLBetweenOperation; +import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLCastOperation; +import sqlancer.mysql.ast.MySQLComputableFunction; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; + +/** + * Recursively applies the {@link EETTransformation EET} transformation rules throughout a MySQL expression's AST. At + * each node the transformer first recurses into (and rebuilds the node from) its transformed children, then, with some + * probability, wraps the resulting sub-expression with a randomly chosen transformation rule. + * + *

+ * A boolean/scalar context flag is threaded through the recursion so that the determined-boolean rules (which reduce an + * expression to a boolean value) are only ever applied where the expression is used purely for its truth value. + */ +public class MySQLEETTransformer { + + private static final boolean BOOLEAN = true; + private static final boolean SCALAR = false; + + private final EETTransformation transformation; + + public MySQLEETTransformer(MySQLExpressionGenerator gen) { + this.transformation = new EETTransformation<>(new MySQLEETNodeFactory(gen)); + } + + /** + * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the + * root, guaranteeing that the returned expression differs from the input. + */ + public MySQLExpression transform(MySQLExpression expr, boolean booleanContext) { + return transformNode(expr, booleanContext, true); + } + + private MySQLExpression transformNode(MySQLExpression expr, boolean booleanContext, boolean forceApply) { + MySQLExpression descended = descend(expr, booleanContext); + if (forceApply || Randomly.getBoolean()) { + return transformation.applyRandomRule(descended, booleanContext); + } + return descended; + } + + /** + * Rebuilds {@code expr} with its children transformed. Leaf nodes (columns, constants, ...) and node types that are + * not rebuilt here are returned unchanged; any applicable transformation is still applied to them by the calling + * {@link #transformNode}. + */ + private MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { + if (expr instanceof MySQLBinaryLogicalOperation) { + // AND/OR/XOR: both operands are evaluated in a boolean context. + MySQLBinaryLogicalOperation op = (MySQLBinaryLogicalOperation) expr; + return new MySQLBinaryLogicalOperation(transformNode(op.getLeft(), BOOLEAN, false), + transformNode(op.getRight(), BOOLEAN, false), op.getOp()); + } else if (expr instanceof MySQLBinaryComparisonOperation) { + MySQLBinaryComparisonOperation op = (MySQLBinaryComparisonOperation) expr; + return new MySQLBinaryComparisonOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLBinaryOperation) { + MySQLBinaryOperation op = (MySQLBinaryOperation) expr; + return new MySQLBinaryOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + MySQLUnaryPrefixOperation op = (MySQLUnaryPrefixOperation) expr; + boolean childContext = op.getOp() == MySQLUnaryPrefixOperator.NOT ? BOOLEAN : SCALAR; + return new MySQLUnaryPrefixOperation(transformNode(op.getExpression(), childContext, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPostfixOperation) { + // The operand is transformed value-preservingly (scalar), which is safe for IS NULL/IS TRUE/IS FALSE. + MySQLUnaryPostfixOperation op = (MySQLUnaryPostfixOperation) expr; + return new MySQLUnaryPostfixOperation(transformNode(op.getExpression(), SCALAR, false), op.getOperator(), + op.isNegated()); + } else if (expr instanceof MySQLCastOperation) { + MySQLCastOperation op = (MySQLCastOperation) expr; + return new MySQLCastOperation(transformNode(op.getExpr(), SCALAR, false), op.getType()); + } else if (expr instanceof MySQLBetweenOperation) { + MySQLBetweenOperation op = (MySQLBetweenOperation) expr; + return new MySQLBetweenOperation(transformNode(op.getExpr(), SCALAR, false), + transformNode(op.getLeft(), SCALAR, false), transformNode(op.getRight(), SCALAR, false)); + } else if (expr instanceof MySQLInOperation) { + MySQLInOperation op = (MySQLInOperation) expr; + List listElements = op.getListElements().stream() + .map(e -> transformNode(e, SCALAR, false)).collect(Collectors.toList()); + return new MySQLInOperation(transformNode(op.getExpr(), SCALAR, false), listElements, op.isTrue()); + } else if (expr instanceof MySQLComputableFunction) { + MySQLComputableFunction op = (MySQLComputableFunction) expr; + MySQLExpression[] args = op.getArguments(); + MySQLExpression[] newArgs = new MySQLExpression[args.length]; + for (int i = 0; i < args.length; i++) { + newArgs[i] = transformNode(args[i], SCALAR, false); + } + return new MySQLComputableFunction(op.getFunction(), newArgs); + } else if (expr instanceof MySQLCaseOperator) { + return descendCase((MySQLCaseOperator) expr); + } + return expr; + } + + private MySQLExpression descendCase(MySQLCaseOperator caseOp) { + MySQLExpression switchCondition = caseOp.getSwitchCondition(); + // Without a switch operand the WHEN conditions are boolean; with one they are compared against the operand. + boolean conditionContext = switchCondition == null ? BOOLEAN : SCALAR; + MySQLExpression newSwitch = switchCondition == null ? null : transformNode(switchCondition, SCALAR, false); + List conditions = caseOp.getConditions().stream() + .map(e -> transformNode(e, conditionContext, false)).collect(Collectors.toList()); + List expressions = caseOp.getExpressions().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); + MySQLExpression elseExpr = caseOp.getElseExpr() == null ? null + : transformNode(caseOp.getElseExpr(), SCALAR, false); + return new MySQLCaseOperator(newSwitch, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index d8ce5dd37..29b8d5ac5 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; @@ -48,11 +49,13 @@ public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, - CERTGenerator { + CERTGenerator, + EETGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; private List tables; + private MySQLEETTransformer eetTransformer; public MySQLExpressionGenerator(MySQLGlobalState state) { this.state = state; @@ -236,6 +239,14 @@ public MySQLSelect generateSelect() { return new MySQLSelect(); } + @Override + public MySQLExpression transformExpression(MySQLExpression expr, boolean booleanContext) { + if (eetTransformer == null) { + eetTransformer = new MySQLEETTransformer(this); + } + return eetTransformer.transform(expr, booleanContext); + } + @Override public List getRandomJoinClauses() { return List.of(); diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 054a66cb6..c17ccf0d5 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -362,7 +362,7 @@ private void appendType(MySQLDataType randomType) { } if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS - || o == MySQLOracleFactory.DQP)) { + || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET)) { sb.append(" ZEROFILL"); } } From 7c9f80f81440213b7a00997e8c0605533fc06b90 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 10 Jul 2026 15:31:17 +0800 Subject: [PATCH 100/132] Clarify that the E in the EET generics must extend the Expression class --- src/sqlancer/common/oracle/EETNodeFactory.java | 4 +++- src/sqlancer/common/oracle/EETTransformation.java | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java index 2cc933ced..c446a6054 100644 --- a/src/sqlancer/common/oracle/EETNodeFactory.java +++ b/src/sqlancer/common/oracle/EETNodeFactory.java @@ -1,5 +1,7 @@ package sqlancer.common.oracle; +import sqlancer.common.ast.newast.Expression; + /** * Factory for constructing the AST nodes needed by the {@link EETTransformation equivalent expression transformation} * rules. Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface @@ -8,7 +10,7 @@ * @param * the DBMS-specific expression type */ -public interface EETNodeFactory { +public interface EETNodeFactory> { /** Builds {@code left AND right}. */ E and(E left, E right); diff --git a/src/sqlancer/common/oracle/EETTransformation.java b/src/sqlancer/common/oracle/EETTransformation.java index ab9b7fefe..ecf2eea74 100644 --- a/src/sqlancer/common/oracle/EETTransformation.java +++ b/src/sqlancer/common/oracle/EETTransformation.java @@ -1,5 +1,6 @@ package sqlancer.common.oracle; +import sqlancer.common.ast.newast.Expression; import sqlancer.Randomly; /** @@ -18,7 +19,7 @@ * @param * the DBMS-specific expression type */ -public class EETTransformation { +public class EETTransformation> { private final EETNodeFactory factory; @@ -39,7 +40,7 @@ private E falseExpr() { } /** - * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen, applicable + * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen applicable * transformation rule. * * @param expr From 95dcc8ef94c5793595ad1a2e5481fbd7411835c2 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 11 Jul 2026 20:56:54 +0800 Subject: [PATCH 101/132] Refactor EET oracle structure for consistency and extensibility Merge `EETTransformation` into a new abstract base class `EETTransformer` (template method pattern: rules + tree-walking orchestration in the base, `descend` abstract for DBMS-specific AST reconstruction). Move `MySQLEETNodeFactory` and `MySQLEETTransformer` from `mysql/gen` to `mysql/oracle` to mirror the placement of their common counterparts. `MySQLEETTransformer` now extends `EETTransformer`. --- .../common/oracle/EETNodeFactory.java | 6 +- .../common/oracle/EETTransformation.java | 84 ------------- .../common/oracle/EETTransformer.java | 112 ++++++++++++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 1 + .../{gen => oracle}/MySQLEETNodeFactory.java | 6 +- .../{gen => oracle}/MySQLEETTransformer.java | 45 ++----- 6 files changed, 128 insertions(+), 126 deletions(-) delete mode 100644 src/sqlancer/common/oracle/EETTransformation.java create mode 100644 src/sqlancer/common/oracle/EETTransformer.java rename src/sqlancer/mysql/{gen => oracle}/MySQLEETNodeFactory.java (95%) rename src/sqlancer/mysql/{gen => oracle}/MySQLEETTransformer.java (72%) diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java index c446a6054..9f97ea937 100644 --- a/src/sqlancer/common/oracle/EETNodeFactory.java +++ b/src/sqlancer/common/oracle/EETNodeFactory.java @@ -3,9 +3,9 @@ import sqlancer.common.ast.newast.Expression; /** - * Factory for constructing the AST nodes needed by the {@link EETTransformation equivalent expression transformation} - * rules. Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface - * lets the (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. + * Factory for constructing the AST nodes needed by the {@link EETTransformer EET transformer's} transformation rules. + * Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface lets the + * (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. * * @param * the DBMS-specific expression type diff --git a/src/sqlancer/common/oracle/EETTransformation.java b/src/sqlancer/common/oracle/EETTransformation.java deleted file mode 100644 index ecf2eea74..000000000 --- a/src/sqlancer/common/oracle/EETTransformation.java +++ /dev/null @@ -1,84 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.common.ast.newast.Expression; -import sqlancer.Randomly; - -/** - * Implements the semantic-preserving expression transformation rules of EET (Equivalent Expression Transformation, Jiang - * & Su, OSDI'24), Table 2. Given an expression, {@link #applyRandomRule} returns a semantically equivalent - * expression built from the primitives provided by an {@link EETNodeFactory}. The rules are DBMS-independent; only the - * node construction (via the factory) is DBMS-specific. - * - *

- * The rules rely on two always-determined boolean expressions built from an arbitrary boolean {@code p}: - *

    - *
  • {@code true_expr(p) = p OR (NOT p) OR (p IS NULL)}, which always evaluates to TRUE, and
    • - *
  • {@code false_expr(p) = p AND (NOT p) AND (p IS NOT NULL)}, which always evaluates to FALSE.
    • - *
- * - * @param - * the DBMS-specific expression type - */ -public class EETTransformation> { - - private final EETNodeFactory factory; - - public EETTransformation(EETNodeFactory factory) { - this.factory = factory; - } - - // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE - private E trueExpr() { - E p = factory.generateBooleanExpression(); - return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); - } - - // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE - private E falseExpr() { - E p = factory.generateBooleanExpression(); - return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); - } - - /** - * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen applicable - * transformation rule. - * - * @param expr - * the expression to transform - * @param booleanContext - * whether {@code expr} is evaluated purely for its truth value (e.g. a WHERE predicate or an operand of a - * logical operator). Only in a boolean context may the determined-boolean rules (No. 1 and 2), which - * reduce the expression to a boolean value, be applied; in a scalar context they would change the - * expression's value/type and are therefore excluded. - * - * @return a semantically equivalent expression - */ - public E applyRandomRule(E expr, boolean booleanContext) { - int rule; - if (booleanContext) { - // Rules No. 1-6 are all value-preserving in a boolean context. - rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); - } else { - if (!factory.isCaseWhenApplicable(expr)) { - return expr; // rule No. 7: transform the expression to itself - } - // In a scalar context only the CASE WHEN rules preserve the exact value and type. - rule = Randomly.fromOptions(3, 4, 5, 6); - } - switch (rule) { - case 1: // bool_expr => false_expr OR bool_expr - return factory.or(falseExpr(), expr); - case 2: // bool_expr => true_expr AND bool_expr - return factory.and(trueExpr(), expr); - case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END - return factory.caseWhen(falseExpr(), expr, expr); - case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END - return factory.caseWhen(trueExpr(), expr, expr); - case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END - case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END - return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); - default: - throw new AssertionError(rule); - } - } -} diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java new file mode 100644 index 000000000..d66935e18 --- /dev/null +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -0,0 +1,112 @@ +package sqlancer.common.oracle; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; + +/** + * Abstract base class for EET (Equivalent Expression Transformation) tree-walkers, based on "Detecting Logic Bugs in + * Database Engines via Equivalent Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework + * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST + * nodes from their transformed children; everything else (the rule logic, context threading, and tree-walking + * orchestration) is provided here. + * + * @param + * the DBMS-specific expression type + */ +public abstract class EETTransformer> { + + private final EETNodeFactory factory; + + protected EETTransformer(EETNodeFactory factory) { + this.factory = factory; + } + + // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE + private E trueExpr() { + E p = factory.generateBooleanExpression(); + return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); + } + + // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE + private E falseExpr() { + E p = factory.generateBooleanExpression(); + return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); + } + + /** + * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent + * expression. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value; only in a boolean context may the + * determined-boolean rules (No. 1 and 2), which reduce the expression to a boolean value, be applied + * + * @return a semantically equivalent expression + */ + protected E applyRandomRule(E expr, boolean booleanContext) { + int rule; + if (booleanContext) { + // Rules No. 1-6 are all value-preserving in a boolean context. + rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); + } else { + if (!factory.isCaseWhenApplicable(expr)) { + return expr; // rule No. 7: transform the expression to itself + } + // In a scalar context only the CASE WHEN rules preserve the exact value and type. + rule = Randomly.fromOptions(3, 4, 5, 6); + } + switch (rule) { + case 1: // bool_expr => false_expr OR bool_expr + return factory.or(falseExpr(), expr); + case 2: // bool_expr => true_expr AND bool_expr + return factory.and(trueExpr(), expr); + case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END + return factory.caseWhen(falseExpr(), expr, expr); + case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END + return factory.caseWhen(trueExpr(), expr, expr); + case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END + case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END + return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); + default: + throw new AssertionError(rule); + } + } + + /** + * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the + * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. + */ + public E transform(E expr, boolean booleanContext) { + return transformNode(expr, booleanContext, true); + } + + /** + * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node. + */ + protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { + E descended = descend(expr, booleanContext); + if (forceApply || Randomly.getBoolean()) { + return applyRandomRule(descended, booleanContext); + } + return descended; + } + + /** + * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each child. + * Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable transformation + * will still be applied to them by the calling {@link #transformNode}. + * + * @param expr + * the expression to descend into + * @param booleanContext + * the context in which {@code expr} itself is evaluated (used to determine child contexts) + * + * @return a rebuilt copy of {@code expr} with transformed children, or {@code expr} itself if it is a leaf + */ + protected abstract E descend(E expr, boolean booleanContext); +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 29b8d5ac5..904013a37 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -46,6 +46,7 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, diff --git a/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java b/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java similarity index 95% rename from src/sqlancer/mysql/gen/MySQLEETNodeFactory.java rename to src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java index 33f3ab587..c73b576c5 100644 --- a/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java +++ b/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java @@ -1,4 +1,4 @@ -package sqlancer.mysql.gen; +package sqlancer.mysql.oracle; import java.util.List; @@ -13,10 +13,10 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.gen.MySQLExpressionGenerator; /** - * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformation EET transformation} - * rules. + * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformer EET transformer}. */ public class MySQLEETNodeFactory implements EETNodeFactory { diff --git a/src/sqlancer/mysql/gen/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java similarity index 72% rename from src/sqlancer/mysql/gen/MySQLEETTransformer.java rename to src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 000e29bb6..71af94ed8 100644 --- a/src/sqlancer/mysql/gen/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -1,10 +1,9 @@ -package sqlancer.mysql.gen; +package sqlancer.mysql.oracle; import java.util.List; import java.util.stream.Collectors; -import sqlancer.Randomly; -import sqlancer.common.oracle.EETTransformation; +import sqlancer.common.oracle.EETTransformer; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -17,49 +16,23 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.gen.MySQLExpressionGenerator; /** - * Recursively applies the {@link EETTransformation EET} transformation rules throughout a MySQL expression's AST. At - * each node the transformer first recurses into (and rebuilds the node from) its transformed children, then, with some - * probability, wraps the resulting sub-expression with a randomly chosen transformation rule. - * - *

- * A boolean/scalar context flag is threaded through the recursion so that the determined-boolean rules (which reduce an - * expression to a boolean value) are only ever applied where the expression is used purely for its truth value. + * MySQL implementation of the {@link EETTransformer EET} tree-walker. Implements {@link #descend} to rebuild MySQL AST + * nodes from their transformed children, threading the correct boolean/scalar context into each child. */ -public class MySQLEETTransformer { +public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; - private final EETTransformation transformation; - public MySQLEETTransformer(MySQLExpressionGenerator gen) { - this.transformation = new EETTransformation<>(new MySQLEETNodeFactory(gen)); - } - - /** - * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the - * root, guaranteeing that the returned expression differs from the input. - */ - public MySQLExpression transform(MySQLExpression expr, boolean booleanContext) { - return transformNode(expr, booleanContext, true); - } - - private MySQLExpression transformNode(MySQLExpression expr, boolean booleanContext, boolean forceApply) { - MySQLExpression descended = descend(expr, booleanContext); - if (forceApply || Randomly.getBoolean()) { - return transformation.applyRandomRule(descended, booleanContext); - } - return descended; + super(new MySQLEETNodeFactory(gen)); } - /** - * Rebuilds {@code expr} with its children transformed. Leaf nodes (columns, constants, ...) and node types that are - * not rebuilt here are returned unchanged; any applicable transformation is still applied to them by the calling - * {@link #transformNode}. - */ - private MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { + @Override + protected MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { if (expr instanceof MySQLBinaryLogicalOperation) { // AND/OR/XOR: both operands are evaluated in a boolean context. MySQLBinaryLogicalOperation op = (MySQLBinaryLogicalOperation) expr; From 657ec00a07e21a523924b7433283f780bbb07aee Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 09:49:12 +0800 Subject: [PATCH 102/132] Move EET transformer ownership to EETOracle, removing EET-specific state from DBMS-specific expression generators --- src/sqlancer/common/gen/EETGenerator.java | 20 +++++++------------ src/sqlancer/common/oracle/EETOracle.java | 6 ++++-- .../mysql/gen/MySQLExpressionGenerator.java | 9 +++------ 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index f468ff6d2..60b91e78a 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -5,14 +5,15 @@ import sqlancer.common.ast.newast.Expression; import sqlancer.common.ast.newast.Join; import sqlancer.common.ast.newast.Select; +import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; /** * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like - * the other oracle generators), an EET generator can transform an expression into a semantically equivalent one - * according to the EET transformation rules. + * the other oracle generators), an EET generator creates a DBMS-specific {@link EETTransformer} that the oracle uses to + * rewrite expressions into semantically equivalent ones. */ public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { @@ -29,16 +30,9 @@ public interface EETGenerator, J extends Join createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 0f192f9ab..0ae229414 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -32,6 +32,7 @@ public class EETOracle, J extends Join, E private final G state; private EETGenerator gen; + private final EETTransformer transformer; private final ExpectedErrors errors; private Reproducer reproducer; @@ -69,6 +70,7 @@ public EETOracle(G state, EETGenerator gen, ExpectedErrors expect } this.state = state; this.gen = gen; + this.transformer = gen.createTransformer(); this.errors = expectedErrors; } @@ -94,10 +96,10 @@ public void check() throws SQLException { // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, // while the WHERE clause is evaluated in a boolean context. - List transformedFetchColumns = fetchColumns.stream().map(c -> gen.transformExpression(c, false)) + List transformedFetchColumns = fetchColumns.stream().map(c -> transformer.transform(c, false)) .collect(Collectors.toList()); select.setFetchColumns(transformedFetchColumns); - select.setWhereClause(gen.transformExpression(whereClause, true)); + select.setWhereClause(transformer.transform(whereClause, true)); String transformedQueryString = select.asString(); List transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 904013a37..f2d577aee 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -46,6 +46,7 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.common.oracle.EETTransformer; import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator @@ -56,7 +57,6 @@ public class MySQLExpressionGenerator extends UntypedExpressionGenerator tables; - private MySQLEETTransformer eetTransformer; public MySQLExpressionGenerator(MySQLGlobalState state) { this.state = state; @@ -241,11 +241,8 @@ public MySQLSelect generateSelect() { } @Override - public MySQLExpression transformExpression(MySQLExpression expr, boolean booleanContext) { - if (eetTransformer == null) { - eetTransformer = new MySQLEETTransformer(this); - } - return eetTransformer.transform(expr, booleanContext); + public EETTransformer createTransformer() { + return new MySQLEETTransformer(this); } @Override From 65957adf9b8fb4b2702c20475c9c5d2e5c972560 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 10:03:19 +0800 Subject: [PATCH 103/132] Rearrange MySQLExpressionGenerator code to group oracle-specific sections together --- .../mysql/gen/MySQLExpressionGenerator.java | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index f2d577aee..5d1ec06f4 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -222,6 +222,22 @@ public List generateOrderBys() { return newOrderBys; } + public MySQLAggregate generateAggregate() { + MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); + + if (func.isVariadic()) { + int nrExprs = Randomly.smallNumber() + 1; + List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) + .collect(Collectors.toList()); + + return new MySQLAggregate(exprs, func); + } else { + return new MySQLAggregate(List.of(generateExpression()), func); + } + } + + // --- Shared oracle infrastructure (TLPWhere / CERT / EET) --- + @Override public MySQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); @@ -240,11 +256,6 @@ public MySQLSelect generateSelect() { return new MySQLSelect(); } - @Override - public EETTransformer createTransformer() { - return new MySQLEETTransformer(this); - } - @Override public List getRandomJoinClauses() { return List.of(); @@ -260,6 +271,8 @@ public List generateFetchColumns(boolean shouldCreateDummy) { return columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); } + // --- CERT oracle --- + @Override public String generateExplainQuery(MySQLSelect select) { return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed @@ -267,20 +280,6 @@ public String generateExplainQuery(MySQLSelect select) { // now be specified } - public MySQLAggregate generateAggregate() { - MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); - - if (func.isVariadic()) { - int nrExprs = Randomly.smallNumber() + 1; - List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) - .collect(Collectors.toList()); - - return new MySQLAggregate(exprs, func); - } else { - return new MySQLAggregate(List.of(generateExpression()), func); - } - } - @Override public boolean mutate(MySQLSelect select) { List> mutators = new ArrayList<>(); @@ -364,4 +363,11 @@ boolean mutateOr(MySQLSelect select) { return true; } } + + // --- EET oracle --- + + @Override + public EETTransformer createTransformer() { + return new MySQLEETTransformer(this); + } } From 03c487476cae5d040de76abff895c1fe5c79ceae Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 10:24:34 +0800 Subject: [PATCH 104/132] Remove need for EETNodeFactory by merging it into EETTransformer --- .../common/oracle/EETNodeFactory.java | 41 ----------- .../common/oracle/EETTransformer.java | 57 ++++++++++----- .../mysql/oracle/MySQLEETNodeFactory.java | 70 ------------------- .../mysql/oracle/MySQLEETTransformer.java | 50 ++++++++++++- 4 files changed, 88 insertions(+), 130 deletions(-) delete mode 100644 src/sqlancer/common/oracle/EETNodeFactory.java delete mode 100644 src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java deleted file mode 100644 index 9f97ea937..000000000 --- a/src/sqlancer/common/oracle/EETNodeFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.common.ast.newast.Expression; - -/** - * Factory for constructing the AST nodes needed by the {@link EETTransformer EET transformer's} transformation rules. - * Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface lets the - * (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. - * - * @param - * the DBMS-specific expression type - */ -public interface EETNodeFactory> { - - /** Builds {@code left AND right}. */ - E and(E left, E right); - - /** Builds {@code left OR right}. */ - E or(E left, E right); - - /** Builds {@code NOT expr}. */ - E not(E expr); - - /** Builds {@code expr IS NULL}. */ - E isNull(E expr); - - /** Builds {@code expr IS NOT NULL}. */ - E isNotNull(E expr); - - /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ - E caseWhen(E condition, E thenExpr, E elseExpr); - - /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ - E generateBooleanExpression(); - - /** - * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not - * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). - */ - boolean isCaseWhenApplicable(E expr); -} diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index d66935e18..010cd0a56 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -10,30 +10,24 @@ *

* This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST - * nodes from their transformed children; everything else (the rule logic, context threading, and tree-walking - * orchestration) is provided here. + * nodes from their transformed children, and the abstract factory methods to construct new nodes; everything else (the + * rule logic, context threading, and tree-walking orchestration) is provided here. * * @param * the DBMS-specific expression type */ public abstract class EETTransformer> { - private final EETNodeFactory factory; - - protected EETTransformer(EETNodeFactory factory) { - this.factory = factory; - } - // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE private E trueExpr() { - E p = factory.generateBooleanExpression(); - return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); + E p = generateBooleanExpression(); + return or(or(p, not(p)), isNull(p)); } // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE private E falseExpr() { - E p = factory.generateBooleanExpression(); - return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); + E p = generateBooleanExpression(); + return and(and(p, not(p)), isNotNull(p)); } /** @@ -54,7 +48,7 @@ protected E applyRandomRule(E expr, boolean booleanContext) { // Rules No. 1-6 are all value-preserving in a boolean context. rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); } else { - if (!factory.isCaseWhenApplicable(expr)) { + if (!isCaseWhenApplicable(expr)) { return expr; // rule No. 7: transform the expression to itself } // In a scalar context only the CASE WHEN rules preserve the exact value and type. @@ -62,16 +56,16 @@ protected E applyRandomRule(E expr, boolean booleanContext) { } switch (rule) { case 1: // bool_expr => false_expr OR bool_expr - return factory.or(falseExpr(), expr); + return or(falseExpr(), expr); case 2: // bool_expr => true_expr AND bool_expr - return factory.and(trueExpr(), expr); + return and(trueExpr(), expr); case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END - return factory.caseWhen(falseExpr(), expr, expr); + return caseWhen(falseExpr(), expr, expr); case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END - return factory.caseWhen(trueExpr(), expr, expr); + return caseWhen(trueExpr(), expr, expr); case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END - return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); + return caseWhen(generateBooleanExpression(), expr, expr); default: throw new AssertionError(rule); } @@ -109,4 +103,31 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * @return a rebuilt copy of {@code expr} with transformed children, or {@code expr} itself if it is a leaf */ protected abstract E descend(E expr, boolean booleanContext); + + /** Builds {@code left AND right}. */ + protected abstract E and(E left, E right); + + /** Builds {@code left OR right}. */ + protected abstract E or(E left, E right); + + /** Builds {@code NOT expr}. */ + protected abstract E not(E expr); + + /** Builds {@code expr IS NULL}. */ + protected abstract E isNull(E expr); + + /** Builds {@code expr IS NOT NULL}. */ + protected abstract E isNotNull(E expr); + + /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ + protected abstract E caseWhen(E condition, E thenExpr, E elseExpr); + + /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ + protected abstract E generateBooleanExpression(); + + /** + * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not + * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + */ + protected abstract boolean isCaseWhenApplicable(E expr); } diff --git a/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java b/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java deleted file mode 100644 index c73b576c5..000000000 --- a/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java +++ /dev/null @@ -1,70 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.util.List; - -import sqlancer.common.oracle.EETNodeFactory; -import sqlancer.mysql.ast.MySQLAggregate; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; -import sqlancer.mysql.ast.MySQLCaseOperator; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLTableReference; -import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; -import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -/** - * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformer EET transformer}. - */ -public class MySQLEETNodeFactory implements EETNodeFactory { - - private final MySQLExpressionGenerator gen; - - public MySQLEETNodeFactory(MySQLExpressionGenerator gen) { - this.gen = gen; - } - - @Override - public MySQLExpression and(MySQLExpression left, MySQLExpression right) { - return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); - } - - @Override - public MySQLExpression or(MySQLExpression left, MySQLExpression right) { - return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); - } - - @Override - public MySQLExpression not(MySQLExpression expr) { - return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); - } - - @Override - public MySQLExpression isNull(MySQLExpression expr) { - return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); - } - - @Override - public MySQLExpression isNotNull(MySQLExpression expr) { - return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); - } - - @Override - public MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { - return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); - } - - @Override - public MySQLExpression generateBooleanExpression() { - return gen.generateBooleanExpression(); - } - - @Override - public boolean isCaseWhenApplicable(MySQLExpression expr) { - // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET - // paper); aggregates are excluded to avoid placing them in invalid contexts. - return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); - } -} diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 71af94ed8..9b0cac226 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -4,16 +4,20 @@ import java.util.stream.Collectors; import sqlancer.common.oracle.EETTransformer; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; import sqlancer.mysql.ast.MySQLBinaryOperation; import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLComputableFunction; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; import sqlancer.mysql.gen.MySQLExpressionGenerator; @@ -27,8 +31,10 @@ public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; + private final MySQLExpressionGenerator gen; + public MySQLEETTransformer(MySQLExpressionGenerator gen) { - super(new MySQLEETNodeFactory(gen)); + this.gen = gen; } @Override @@ -94,4 +100,46 @@ private MySQLExpression descendCase(MySQLCaseOperator caseOp) { : transformNode(caseOp.getElseExpr(), SCALAR, false); return new MySQLCaseOperator(newSwitch, conditions, expressions, elseExpr); } + + @Override + protected MySQLExpression and(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); + } + + @Override + protected MySQLExpression or(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); + } + + @Override + protected MySQLExpression not(MySQLExpression expr) { + return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); + } + + @Override + protected MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); + } + + @Override + protected MySQLExpression isNotNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); + } + + @Override + protected MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { + return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); + } + + @Override + protected MySQLExpression generateBooleanExpression() { + return gen.generateBooleanExpression(); + } + + @Override + protected boolean isCaseWhenApplicable(MySQLExpression expr) { + // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET + // paper); aggregates are excluded to avoid placing them in invalid contexts. + return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); + } } From 9285e9168dd3a3d06cc080b6cc75d46aea216529 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 18:17:44 +0800 Subject: [PATCH 105/132] Correct EET transformation rule comments --- src/sqlancer/common/oracle/EETTransformer.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index 010cd0a56..12999402c 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -55,17 +55,18 @@ protected E applyRandomRule(E expr, boolean booleanContext) { rule = Randomly.fromOptions(3, 4, 5, 6); } switch (rule) { - case 1: // bool_expr => false_expr OR bool_expr + case 1: // expr => false_expr OR expr return or(falseExpr(), expr); - case 2: // bool_expr => true_expr AND bool_expr + case 2: // expr => true_expr AND expr return and(trueExpr(), expr); - case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END + case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END return caseWhen(falseExpr(), expr, expr); - case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END + case 4: // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END return caseWhen(trueExpr(), expr, expr); - case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END - case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END - return caseWhen(generateBooleanExpression(), expr, expr); + case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END + case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END + return caseWhen(generateBooleanExpression(), expr, expr); + // deep copy of expr is not needed, as the AST nodes are immutable anyway default: throw new AssertionError(rule); } From 52ce86661581681b1eb754ebcdc89103c833fa76 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 13 Jul 2026 12:40:14 +0800 Subject: [PATCH 106/132] Implement rules 3 and 4 for EET --- src/sqlancer/common/gen/EETGenerator.java | 2 +- src/sqlancer/common/oracle/EETOracle.java | 2 +- .../common/oracle/EETTransformer.java | 51 ++++++- .../mysql/ast/MySQLCastOperation.java | 7 +- .../mysql/gen/MySQLExpressionGenerator.java | 2 +- .../mysql/oracle/MySQLEETTransformer.java | 128 +++++++++++++++++- 6 files changed, 179 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index 60b91e78a..85aaa7462 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -34,5 +34,5 @@ public interface EETGenerator, J extends Join createTransformer(); + EETTransformer createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 0ae229414..148532e9b 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -32,7 +32,7 @@ public class EETOracle, J extends Join, E private final G state; private EETGenerator gen; - private final EETTransformer transformer; + private final EETTransformer transformer; private final ExpectedErrors errors; private Reproducer reproducer; diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index 12999402c..f27a917b9 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -10,13 +10,16 @@ *

* This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST - * nodes from their transformed children, and the abstract factory methods to construct new nodes; everything else (the - * rule logic, context threading, and tree-walking orchestration) is provided here. + * nodes from their transformed children, the abstract factory methods to construct new nodes, and the type hooks + * ({@link #inferType} and {@link #generateExpressionOfType}) that realize the paper's {@code rand_expr(type(expr))}; + * everything else (the rule logic, context threading, and tree-walking orchestration) is provided here. * * @param - * the DBMS-specific expression type + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain used by {@link #inferType} and {@link #generateExpressionOfType} */ -public abstract class EETTransformer> { +public abstract class EETTransformer, T> { // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE private E trueExpr() { @@ -30,6 +33,22 @@ private E falseExpr() { return and(and(p, not(p)), isNotNull(p)); } + /** + * Implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches that of + * {@code expr}. Although the generated expression is never evaluated (it occupies the dead branch of rules No. 3 + * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could + * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to + * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form + * of rules No. 5 and 6). + */ + private E randExprOfSameType(E expr) { + T type = inferType(expr); + if (type == null) { + return expr; + } + return generateExpressionOfType(type); + } + /** * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent * expression. @@ -60,12 +79,12 @@ protected E applyRandomRule(E expr, boolean booleanContext) { case 2: // expr => true_expr AND expr return and(trueExpr(), expr); case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END - return caseWhen(falseExpr(), expr, expr); + return caseWhen(falseExpr(), randExprOfSameType(expr), expr); case 4: // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END - return caseWhen(trueExpr(), expr, expr); + return caseWhen(trueExpr(), expr, randExprOfSameType(expr)); case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END - return caseWhen(generateBooleanExpression(), expr, expr); + return caseWhen(generateBooleanExpression(), expr, expr); // deep copy of expr is not needed, as the AST nodes are immutable anyway default: throw new AssertionError(rule); @@ -126,6 +145,24 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ protected abstract E generateBooleanExpression(); + /** + * Infers the static type of {@code expr}, or returns {@code null} if it cannot be determined. The type domain + * {@code T} is DBMS-specific and may be coarse: it only needs to be precise enough that replacing an expression + * with another of the same {@code T} leaves the DBMS's CASE WHEN result-type resolution unaffected. Returning + * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead + * branch. Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is + * uncertain. + */ + protected abstract T inferType(E expr); + + /** + * Generates a fresh random expression of static type {@code type}, reusing the variables available to the query + * generator. DBMSs with a typed expression generator can delegate to it directly; DBMSs with an untyped generator + * can instead wrap an arbitrary random expression in a CAST to {@code type} (which requires every value of + * {@code T} to be a valid CAST target). + */ + protected abstract E generateExpressionOfType(T type); + /** * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index 8ae783fa0..8a457d0f1 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -6,11 +6,14 @@ public class MySQLCastOperation implements MySQLExpression { private final CastType type; public enum CastType { - SIGNED, UNSIGNED; + SIGNED, UNSIGNED, + // CHAR and DOUBLE are used only by the EET oracle's type-pinning casts and are never evaluated, so + // MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + CHAR, DOUBLE; public static CastType getRandom() { return SIGNED; - // return Randomly.fromOptions(CastType.values()); + // return Randomly.fromOptions(CastType.SIGNED, CastType.UNSIGNED); } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 5d1ec06f4..9b019a6d4 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -367,7 +367,7 @@ boolean mutateOr(MySQLSelect select) { // --- EET oracle --- @Override - public EETTransformer createTransformer() { + public EETTransformer createTransformer() { return new MySQLEETTransformer(this); } } diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 9b0cac226..d16a8e517 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -1,5 +1,6 @@ package sqlancer.mysql.oracle; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -12,7 +13,11 @@ import sqlancer.mysql.ast.MySQLBinaryOperation; import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; +import sqlancer.mysql.ast.MySQLCastOperation.CastType; +import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLComputableFunction; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; import sqlancer.mysql.ast.MySQLTableReference; @@ -25,8 +30,13 @@ /** * MySQL implementation of the {@link EETTransformer EET} tree-walker. Implements {@link #descend} to rebuild MySQL AST * nodes from their transformed children, threading the correct boolean/scalar context into each child. + * + *

+ * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target types + * ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning {@code null} + * when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping it in a CAST. */ -public class MySQLEETTransformer extends EETTransformer { +public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; @@ -136,6 +146,122 @@ protected MySQLExpression generateBooleanExpression() { return gen.generateBooleanExpression(); } + @Override + protected MySQLExpression generateExpressionOfType(CastType type) { + // The MySQL expression generator is untyped, so the type of an arbitrary random expression is pinned by + // wrapping it in a CAST to the requested type. + return new MySQLCastOperation(gen.generateExpression(), type); + } + + @Override + protected CastType inferType(MySQLExpression expr) { + if (expr instanceof MySQLBinaryLogicalOperation || expr instanceof MySQLBinaryComparisonOperation + || expr instanceof MySQLUnaryPostfixOperation || expr instanceof MySQLBetweenOperation + || expr instanceof MySQLInOperation || expr instanceof MySQLExists) { + // Predicates evaluate to the boolean values 0/1, which are signed BIGINT. + return CastType.SIGNED; + } else if (expr instanceof MySQLBinaryOperation) { + // The bit operators &, | and ^ return BIGINT UNSIGNED. + return CastType.UNSIGNED; + } else if (expr instanceof MySQLCastOperation) { + return ((MySQLCastOperation) expr).getType(); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + return inferUnaryPrefixType((MySQLUnaryPrefixOperation) expr); + } else if (expr instanceof MySQLConstant) { + return inferConstantType((MySQLConstant) expr); + } else if (expr instanceof MySQLColumnReference) { + return inferColumnType((MySQLColumnReference) expr); + } else if (expr instanceof MySQLComputableFunction) { + return inferFunctionType((MySQLComputableFunction) expr); + } else if (expr instanceof MySQLCaseOperator) { + return inferCaseType((MySQLCaseOperator) expr); + } + return null; + } + + private CastType inferUnaryPrefixType(MySQLUnaryPrefixOperation op) { + if (op.getOp() == MySQLUnaryPrefixOperator.NOT) { + return CastType.SIGNED; + } + CastType operandType = inferType(op.getExpression()); + if (op.getOp() == MySQLUnaryPrefixOperator.PLUS) { + return operandType; // unary + is the identity + } + // Unary -: strings are coerced to DOUBLE; negating UNSIGNED changes the type (and usually errors). + if (operandType == CastType.CHAR || operandType == CastType.DOUBLE) { + return CastType.DOUBLE; + } + return operandType == CastType.SIGNED ? CastType.SIGNED : null; + } + + private CastType inferConstantType(MySQLConstant constant) { + if (constant instanceof MySQLConstant.MySQLIntConstant) { + return constant.isSigned() ? CastType.SIGNED : CastType.UNSIGNED; + } else if (constant instanceof MySQLConstant.MySQLTextConstant) { + return CastType.CHAR; + } else if (constant instanceof MySQLConstant.MySQLDoubleConstant) { + return CastType.DOUBLE; + } + return null; // the NULL constant has no type of its own + } + + private CastType inferColumnType(MySQLColumnReference ref) { + switch (ref.getColumn().getType()) { + case INT: + return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns + case VARCHAR: + return CastType.CHAR; + case DOUBLE: + return CastType.DOUBLE; + case FLOAT: // FLOAT-to-DOUBLE widening in the CASE result changes the rendered value + case DECIMAL: // the CASE result would need the column's exact precision and scale + default: + return null; + } + } + + private CastType inferFunctionType(MySQLComputableFunction func) { + MySQLExpression[] args = func.getArguments(); + switch (func.getFunction()) { + case BIT_COUNT: + return CastType.SIGNED; + case IF: + // The result type aggregates the types of the two value arguments (the condition does not contribute). + return commonType(args[1], args[2]); + case COALESCE: + case IFNULL: + case LEAST: + case GREATEST: + return commonType(args); + default: + return null; + } + } + + private CastType inferCaseType(MySQLCaseOperator caseOp) { + List branches = new ArrayList<>(caseOp.getExpressions()); + if (caseOp.getElseExpr() != null) { + branches.add(caseOp.getElseExpr()); + } + return commonType(branches.toArray(new MySQLExpression[0])); + } + + /** + * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the + * same inferrable type (a conservative under-approximation of MySQL's aggregation rules). + */ + private CastType commonType(MySQLExpression... exprs) { + CastType common = null; + for (MySQLExpression expr : exprs) { + CastType type = inferType(expr); + if (type == null || common != null && type != common) { + return null; + } + common = type; + } + return common; + } + @Override protected boolean isCaseWhenApplicable(MySQLExpression expr) { // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET From 86e5b719a6b7fffc548e37ad15a6dd54657797c4 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 15 Jul 2026 09:50:26 +0800 Subject: [PATCH 107/132] Fix EET reproducer/reduction to port the same fixes to it that were earlier made to TLP WHERE --- src/sqlancer/common/oracle/EETOracle.java | 92 +++++++++++++++++++---- 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 148532e9b..5af0cd1b3 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -40,28 +40,73 @@ public class EETOracle, J extends Join, E private final class EETReproducer implements Reproducer { private final String originalQueryString; + // null if the original bug was a DBMS error on the original query alone private final String transformedQueryString; - private final List resultSet; + // null if the original bug is a result set mismatch; otherwise, the message of the + // unexpected DBMS error that the original or transformed query triggered + private final String expectedErrorMessage; - EETReproducer(String originalQueryString, String transformedQueryString, List resultSet) { + EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; - this.resultSet = resultSet; + this.expectedErrorMessage = expectedErrorMessage; } @Override public boolean bugStillTriggers(G globalState) { + List originalResultSet; + List transformedResultSet; try { - List transformedResultSet = ComparatorHelper - .getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, transformedResultSet, originalQueryString, + // Re-execute both queries against the current (reduced) database instead of comparing + // against a cached result set, which would be stale once statements have been removed. + originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + globalState); + if (transformedQueryString == null) { + // the original bug was a DBMS error on the original query alone, which no + // longer occurs + return false; + } + transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, + globalState); + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs + return false; + } + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), globalState); - } catch (AssertionError triggeredError) { + } catch (AssertionError resultSetMismatch) { return true; - } catch (SQLException ignored) { } return false; } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (transformedQueryString != null) { + sb.append("-- transformed: ").append(transformedQueryString).append(';') + .append(System.lineSeparator()); + } + return sb.toString(); + } } public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { @@ -91,8 +136,16 @@ public void check() throws SQLException { String originalQueryString = select.asString(); generatedQueryString = originalQueryString; - List originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - state); + List originalResultSet; + try { + originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + } catch (AssertionError unexpectedError) { + // an unexpected DBMS error on the original query alone is itself a bug worth reducing; + // transformedQueryString is null because no transformed query is involved + reproducer = new EETReproducer(originalQueryString, null, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, // while the WHERE clause is evaluated in a boolean context. @@ -102,13 +155,24 @@ public void check() throws SQLException { select.setWhereClause(transformer.transform(whereClause, true)); String transformedQueryString = select.asString(); - List transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, - errors, state); + List transformedResultSet; + try { + transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, + state); + } catch (AssertionError unexpectedError) { + // the semantics-preserving transformation made the query trigger a DBMS error that the + // original did not, which is a bug worth reducing + reproducer = new EETReproducer(originalQueryString, transformedQueryString, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + // Set the reproducer before the assertion: assumeResultSetsAreEqual throws when the bug is + // detected, so creating the reproducer afterwards would leave it null and prevent any reduction. + reproducer = new EETReproducer(originalQueryString, transformedQueryString, null); ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), state); - - reproducer = new EETReproducer(originalQueryString, transformedQueryString, originalResultSet); } @Override From 912197dba19e2f6321d9881cd815172c779dec73 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 23 Jul 2026 15:48:24 +0800 Subject: [PATCH 108/132] Add FLOAT and DECIMAL to MySQL EET target cast types --- src/sqlancer/common/gen/EETGenerator.java | 6 +++--- src/sqlancer/common/oracle/EETOracle.java | 9 ++++----- .../common/oracle/EETTransformer.java | 16 +++++++-------- .../mysql/ast/MySQLCastOperation.java | 6 +++--- .../mysql/oracle/MySQLEETTransformer.java | 20 +++++++++++++------ 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index 85aaa7462..ef436d975 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -30,9 +30,9 @@ public interface EETGenerator, J extends Join createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 5af0cd1b3..1422d5b6b 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -23,9 +23,9 @@ * *

* The oracle generates a random query and then transforms its expressions (the WHERE predicate and the fetch columns) - * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation preserves - * semantics, the original and the transformed query must return the same result set; any discrepancy indicates a logic - * bug in the DBMS. + * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation + * preserves semantics, the original and the transformed query must return the same result set; any discrepancy + * indicates a logic bug in the DBMS. */ public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> implements TestOracle { @@ -102,8 +102,7 @@ public String getBugInformation() { } sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); if (transformedQueryString != null) { - sb.append("-- transformed: ").append(transformedQueryString).append(';') - .append(System.lineSeparator()); + sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); } return sb.toString(); } diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index f27a917b9..932a6fb8e 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -38,8 +38,8 @@ private E falseExpr() { * {@code expr}. Although the generated expression is never evaluated (it occupies the dead branch of rules No. 3 * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to - * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form - * of rules No. 5 and 6). + * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form of + * rules No. 5 and 6). */ private E randExprOfSameType(E expr) { T type = inferType(expr); @@ -85,7 +85,7 @@ protected E applyRandomRule(E expr, boolean booleanContext) { case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END return caseWhen(generateBooleanExpression(), expr, expr); - // deep copy of expr is not needed, as the AST nodes are immutable anyway + // deep copy of expr is not needed, as the AST nodes are immutable anyway default: throw new AssertionError(rule); } @@ -111,9 +111,9 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { } /** - * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each child. - * Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable transformation - * will still be applied to them by the calling {@link #transformNode}. + * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each + * child. Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable + * transformation will still be applied to them by the calling {@link #transformNode}. * * @param expr * the expression to descend into @@ -149,8 +149,8 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * Infers the static type of {@code expr}, or returns {@code null} if it cannot be determined. The type domain * {@code T} is DBMS-specific and may be coarse: it only needs to be precise enough that replacing an expression * with another of the same {@code T} leaves the DBMS's CASE WHEN result-type resolution unaffected. Returning - * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead - * branch. Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is + * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead branch. + * Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is * uncertain. */ protected abstract T inferType(E expr); diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index 8a457d0f1..b71d0498c 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -7,9 +7,9 @@ public class MySQLCastOperation implements MySQLExpression { public enum CastType { SIGNED, UNSIGNED, - // CHAR and DOUBLE are used only by the EET oracle's type-pinning casts and are never evaluated, so - // MySQLConstant.castAs does not support them; they must not be returned by getRandom(). - CHAR, DOUBLE; + // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never + // evaluated, so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + CHAR, FLOAT, DOUBLE, DECIMAL; public static CastType getRandom() { return SIGNED; diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index d16a8e517..5972996b8 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -80,8 +80,8 @@ protected MySQLExpression descend(MySQLExpression expr, boolean booleanContext) transformNode(op.getLeft(), SCALAR, false), transformNode(op.getRight(), SCALAR, false)); } else if (expr instanceof MySQLInOperation) { MySQLInOperation op = (MySQLInOperation) expr; - List listElements = op.getListElements().stream() - .map(e -> transformNode(e, SCALAR, false)).collect(Collectors.toList()); + List listElements = op.getListElements().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); return new MySQLInOperation(transformNode(op.getExpr(), SCALAR, false), listElements, op.isTrue()); } else if (expr instanceof MySQLComputableFunction) { MySQLComputableFunction op = (MySQLComputableFunction) expr; @@ -211,10 +211,18 @@ private CastType inferColumnType(MySQLColumnReference ref) { return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns case VARCHAR: return CastType.CHAR; + case FLOAT: + // Assumes FLOAT columns are never created with (M, D); otherwise the CAST would need the exact + // precision/scale. + return CastType.FLOAT; case DOUBLE: + // Assumes DOUBLE columns are never created with (M, D); otherwise the CAST would need the exact + // precision/scale. return CastType.DOUBLE; - case FLOAT: // FLOAT-to-DOUBLE widening in the CASE result changes the rendered value - case DECIMAL: // the CASE result would need the column's exact precision and scale + case DECIMAL: + // Assumes DECIMAL columns are never created with (M, D); otherwise the CAST would need the exact + // precision/scale. + return CastType.DECIMAL; default: return null; } @@ -247,8 +255,8 @@ private CastType inferCaseType(MySQLCaseOperator caseOp) { } /** - * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the - * same inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same + * inferrable type (a conservative under-approximation of MySQL's aggregation rules). */ private CastType commonType(MySQLExpression... exprs) { CastType common = null; From 4d6a651d5da612c26b4c180796b174c1fddc2dfc Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 23 Jul 2026 17:38:12 +0800 Subject: [PATCH 109/132] Fix treatment of unary minus in MySQL EET implementation --- .../mysql/oracle/MySQLEETTransformer.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 5972996b8..eb107c94b 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -185,13 +185,18 @@ private CastType inferUnaryPrefixType(MySQLUnaryPrefixOperation op) { } CastType operandType = inferType(op.getExpression()); if (op.getOp() == MySQLUnaryPrefixOperator.PLUS) { - return operandType; // unary + is the identity + return operandType; } - // Unary -: strings are coerced to DOUBLE; negating UNSIGNED changes the type (and usually errors). - if (operandType == CastType.CHAR || operandType == CastType.DOUBLE) { - return CastType.DOUBLE; + if (op.getOp() == MySQLUnaryPrefixOperator.MINUS) { + if (operandType == CastType.UNSIGNED) { + return CastType.SIGNED; + } else if (operandType == CastType.FLOAT) { + return CastType.DOUBLE; + } else if (operandType != CastType.CHAR) { + return operandType; + } } - return operandType == CastType.SIGNED ? CastType.SIGNED : null; + return null; } private CastType inferConstantType(MySQLConstant constant) { From e3d9f61ec8e09310aed7e8b686d777116c36776a Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 26 Jul 2026 15:02:27 +0800 Subject: [PATCH 110/132] Restrict creation of (M, D) columns to prevent false positives in EET --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 9 +++++++-- .../mysql/oracle/MySQLEETTransformer.java | 16 +++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index c17ccf0d5..40e325041 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -368,8 +368,13 @@ private void appendType(MySQLDataType randomType) { } } - public static void optionallyAddPrecisionAndScale(StringBuilder sb) { - if (Randomly.getBoolean() && !MySQLBugs.bug99183) { + private void optionallyAddPrecisionAndScale(StringBuilder sb) { + // The EET oracle's type inference assumes FLOAT/DOUBLE/DECIMAL columns are created without (M, D) (see + // MySQLEETTransformer#inferColumnType), so precision/scale is omitted while EET is active. This restriction can + // be lifted once (M, D) is tracked through the codebase and reflected in the CAST target types. + boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.EET); + if (Randomly.getBoolean() && !MySQLBugs.bug99183 && !eetActive) { sb.append("("); // The maximum number of digits (M) for DECIMAL is 65 long m = Randomly.getNotCachedInteger(1, 65); diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index eb107c94b..27a62869a 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -32,9 +32,10 @@ * nodes from their transformed children, threading the correct boolean/scalar context into each child. * *

- * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target types - * ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning {@code null} - * when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping it in a CAST. + * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target + * types ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning + * {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping + * it in a CAST. */ public class MySQLEETTransformer extends EETTransformer { @@ -216,17 +217,14 @@ private CastType inferColumnType(MySQLColumnReference ref) { return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns case VARCHAR: return CastType.CHAR; + // FLOAT/DOUBLE/DECIMAL columns are created without (M, D) while EET is active, so the plain + // CAST target below matches the column's type. Reintroducing (M, D) for better coverage would + // require tracking it here and emitting the exact precision/scale in the CAST. case FLOAT: - // Assumes FLOAT columns are never created with (M, D); otherwise the CAST would need the exact - // precision/scale. return CastType.FLOAT; case DOUBLE: - // Assumes DOUBLE columns are never created with (M, D); otherwise the CAST would need the exact - // precision/scale. return CastType.DOUBLE; case DECIMAL: - // Assumes DECIMAL columns are never created with (M, D); otherwise the CAST would need the exact - // precision/scale. return CastType.DECIMAL; default: return null; From 0a09fba83235c0b129e10a11538e3835ee1a7a1c Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 26 Jul 2026 15:15:33 +0800 Subject: [PATCH 111/132] Fix checkstyle violations --- src/sqlancer/common/gen/EETGenerator.java | 13 +++ src/sqlancer/common/oracle/EETOracle.java | 15 +++ .../common/oracle/EETTransformer.java | 110 ++++++++++++++++-- .../mysql/gen/MySQLExpressionGenerator.java | 2 +- .../mysql/oracle/MySQLEETTransformer.java | 7 +- 5 files changed, 135 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index ef436d975..a87a635ec 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -14,6 +14,17 @@ * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like * the other oracle generators), an EET generator creates a DBMS-specific {@link EETTransformer} that the oracle uses to * rewrite expressions into semantically equivalent ones. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class */ public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { @@ -33,6 +44,8 @@ public interface EETGenerator, J extends Join createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 1422d5b6b..b5aedf2cb 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -26,6 +26,21 @@ * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation * preserves semantics, the original and the transformed query must return the same result set; any discrepancy * indicates a logic bug in the DBMS. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific schema class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + * @param + * the DBMS-specific global state class */ public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> implements TestOracle { diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index 932a6fb8e..c46e148e1 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -24,7 +24,7 @@ public abstract class EETTransformer, T> { // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE private E trueExpr() { E p = generateBooleanExpression(); - return or(or(p, not(p)), isNull(p)); + return orExpr(orExpr(p, not(p)), isNull(p)); } // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE @@ -40,6 +40,11 @@ private E falseExpr() { * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form of * rules No. 5 and 6). + * + * @param expr + * the expression whose static type the generated expression must match + * + * @return a random expression whose static type matches that of {@code expr} */ private E randExprOfSameType(E expr) { T type = inferType(expr); @@ -75,7 +80,7 @@ protected E applyRandomRule(E expr, boolean booleanContext) { } switch (rule) { case 1: // expr => false_expr OR expr - return or(falseExpr(), expr); + return orExpr(falseExpr(), expr); case 2: // expr => true_expr AND expr return and(trueExpr(), expr); case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END @@ -94,6 +99,13 @@ protected E applyRandomRule(E expr, boolean booleanContext) { /** * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * + * @return a semantically equivalent expression */ public E transform(E expr, boolean booleanContext) { return transformNode(expr, booleanContext, true); @@ -101,6 +113,15 @@ public E transform(E expr, boolean booleanContext) { /** * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * @param forceApply + * whether a rule must be applied at this node rather than only with some probability + * + * @return the transformed expression */ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { E descended = descend(expr, booleanContext); @@ -124,25 +145,79 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { */ protected abstract E descend(E expr, boolean booleanContext); - /** Builds {@code left AND right}. */ + /** + * Builds {@code left AND right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left AND right} expression + */ protected abstract E and(E left, E right); - /** Builds {@code left OR right}. */ - protected abstract E or(E left, E right); + /** + * Builds {@code left OR right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left OR right} expression + */ + protected abstract E orExpr(E left, E right); - /** Builds {@code NOT expr}. */ + /** + * Builds {@code NOT expr}. + * + * @param expr + * the operand + * + * @return the {@code NOT expr} expression + */ protected abstract E not(E expr); - /** Builds {@code expr IS NULL}. */ + /** + * Builds {@code expr IS NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NULL} expression + */ protected abstract E isNull(E expr); - /** Builds {@code expr IS NOT NULL}. */ + /** + * Builds {@code expr IS NOT NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NOT NULL} expression + */ protected abstract E isNotNull(E expr); - /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ + /** + * Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. + * + * @param condition + * the WHEN condition + * @param thenExpr + * the THEN branch + * @param elseExpr + * the ELSE branch + * + * @return the CASE WHEN expression + */ protected abstract E caseWhen(E condition, E thenExpr, E elseExpr); - /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ + /** + * Generates a fresh random boolean expression, reusing the variables available to the query generator. + * + * @return a fresh random boolean expression + */ protected abstract E generateBooleanExpression(); /** @@ -152,6 +227,11 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead branch. * Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is * uncertain. + * + * @param expr + * the expression whose static type is to be inferred + * + * @return the inferred static type of {@code expr}, or {@code null} if it cannot be determined */ protected abstract T inferType(E expr); @@ -160,12 +240,22 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * generator. DBMSs with a typed expression generator can delegate to it directly; DBMSs with an untyped generator * can instead wrap an arbitrary random expression in a CAST to {@code type} (which requires every value of * {@code T} to be a valid CAST target). + * + * @param type + * the static type the generated expression must have + * + * @return a fresh random expression of static type {@code type} */ protected abstract E generateExpressionOfType(T type); /** * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + * + * @param expr + * the expression to test + * + * @return {@code true} if {@code expr} may be wrapped in a CASE WHEN expression */ protected abstract boolean isCaseWhenApplicable(E expr); } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9b019a6d4..da304ac67 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -12,6 +12,7 @@ import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTables; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; @@ -46,7 +47,6 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; -import sqlancer.common.oracle.EETTransformer; import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 27a62869a..2e4394eec 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -118,7 +118,7 @@ protected MySQLExpression and(MySQLExpression left, MySQLExpression right) { } @Override - protected MySQLExpression or(MySQLExpression left, MySQLExpression right) { + protected MySQLExpression orExpr(MySQLExpression left, MySQLExpression right) { return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); } @@ -260,6 +260,11 @@ private CastType inferCaseType(MySQLCaseOperator caseOp) { /** * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same * inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * + * @param exprs + * the result-type-determining subexpressions + * + * @return the common inferred type of {@code exprs}, or {@code null} if they do not share one */ private CastType commonType(MySQLExpression... exprs) { CastType common = null; From c71d198fff6cb271e1f946addbe5c43597580636 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 13:09:02 +0800 Subject: [PATCH 112/132] Refactor EETTransformer to encode the transformation rules as an enum --- .../common/oracle/EETTransformer.java | 159 ++++++++++++++---- 1 file changed, 129 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index c46e148e1..b472b8aff 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -1,5 +1,8 @@ package sqlancer.common.oracle; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Expression; @@ -35,11 +38,10 @@ private E falseExpr() { /** * Implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches that of - * {@code expr}. Although the generated expression is never evaluated (it occupies the dead branch of rules No. 3 + * {@code expr}. Although the generated expression is never evaluated (it occupies the redundant branch of rules 3 * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to - * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form of - * rules No. 5 and 6). + * {@code expr} itself, which trivially has the correct type (degenerating to rules 5 and 6). * * @param expr * the expression whose static type the generated expression must match @@ -54,46 +56,143 @@ private E randExprOfSameType(E expr) { return generateExpressionOfType(type); } + /** + * The first six transformation rules of the EET paper (Table 2). Each rule knows how to apply itself + * ({@link #apply}) and in which contexts it preserves the expression's value ({@link #isApplicable}). Rule No. 7 + * (transform the expression to itself) is not modelled here: it is the fallback applied by {@link #applyRandomRule} + * when no other rule is applicable. + */ + private enum Rule { + // expr => false_expr OR expr + RULE_1 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.orExpr(t.falseExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => true_expr AND expr + RULE_2 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.and(t.trueExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END + RULE_3 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.falseExpr(), t.randExprOfSameType(expr), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END + RULE_4 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.trueExpr(), expr, t.randExprOfSameType(expr)); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END + RULE_5 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END + RULE_6 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }; + + /** + * Applies this rule to {@code expr}, producing a semantically equivalent expression. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain + * @param t + * the transformer providing the DBMS-specific node factories + * @param expr + * the expression to transform + * + * @return a semantically equivalent expression + */ + abstract , T> E apply(EETTransformer t, E expr); + + /** + * Whether this rule preserves {@code expr}'s value in the given context. + * + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (rules 1 and 2 are only applicable if + * this is the case) + * @param caseWhenApplicable + * whether {@code expr} may be wrapped in a CASE WHEN expression + * + * @return {@code true} if this rule preserves {@code expr}'s value in the given context + */ + abstract boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable); + } + /** * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent - * expression. + * expression. When no rule is applicable, {@code expr} is returned unchanged (rule No. 7 of the EET paper). * * @param expr * the expression to transform * @param booleanContext - * whether {@code expr} is evaluated purely for its truth value; only in a boolean context may the - * determined-boolean rules (No. 1 and 2), which reduce the expression to a boolean value, be applied + * whether {@code expr} is evaluated purely for its truth value * * @return a semantically equivalent expression */ protected E applyRandomRule(E expr, boolean booleanContext) { - int rule; - if (booleanContext) { - // Rules No. 1-6 are all value-preserving in a boolean context. - rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); - } else { - if (!isCaseWhenApplicable(expr)) { - return expr; // rule No. 7: transform the expression to itself + boolean caseWhenApplicable = isCaseWhenApplicable(expr); + List applicableRules = new ArrayList<>(); + for (Rule rule : Rule.values()) { + if (rule.isApplicable(booleanContext, caseWhenApplicable)) { + applicableRules.add(rule); } - // In a scalar context only the CASE WHEN rules preserve the exact value and type. - rule = Randomly.fromOptions(3, 4, 5, 6); } - switch (rule) { - case 1: // expr => false_expr OR expr - return orExpr(falseExpr(), expr); - case 2: // expr => true_expr AND expr - return and(trueExpr(), expr); - case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END - return caseWhen(falseExpr(), randExprOfSameType(expr), expr); - case 4: // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END - return caseWhen(trueExpr(), expr, randExprOfSameType(expr)); - case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END - case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END - return caseWhen(generateBooleanExpression(), expr, expr); - // deep copy of expr is not needed, as the AST nodes are immutable anyway - default: - throw new AssertionError(rule); + if (applicableRules.isEmpty()) { + return expr; // rule 7 fallback: transform expression to itself } + return Randomly.fromList(applicableRules).apply(this, expr); } /** From 29291c7a7c73eafd2621240bec3da4d8db4be6dd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 14:35:44 +0800 Subject: [PATCH 113/132] Factor out reproducer logic for reuse among multiple different test oracles --- .../oracle/AbstractComparisonReproducer.java | 142 ++++++++++++++++++ src/sqlancer/common/oracle/EETOracle.java | 72 ++++----- src/sqlancer/common/oracle/NoRECOracle.java | 62 ++++---- .../common/oracle/TLPWhereOracle.java | 85 ++++++----- 4 files changed, 239 insertions(+), 122 deletions(-) create mode 100644 src/sqlancer/common/oracle/AbstractComparisonReproducer.java diff --git a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java new file mode 100644 index 000000000..d20a9636a --- /dev/null +++ b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java @@ -0,0 +1,142 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; + +/** + * Shared skeleton for the {@link Reproducer}s of oracles that detect a bug by comparing two evaluations of a + * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}. All of these + * reduce the bug the same way: re-evaluate both sides against the reduced database and report whether they still + * disagree (or, when the original bug was an unexpected DBMS error, whether that same error still fires). + * + *

+ * This class owns that control flow (including distinguishing a still-reproducing error from an unrelated one + * introduced by the reduction) and the {@link #getBugInformation()} header. Subclasses supply the parts specific to + * their oracle: how each side is evaluated, how the two are compared, and how the failing queries are rendered in the + * reduced test case. + * + * @param + * the DBMS-specific global state class + * @param + * the type each side evaluates to (e.g. a result set as a list of strings, a row count, a post-image) + */ +public abstract class AbstractComparisonReproducer, R> implements Reproducer { + + /** + * The message of the unexpected DBMS error the original bug was, or {@code null} if the original bug was a + * comparison mismatch rather than an error. + */ + protected final String expectedErrorMessage; + + protected AbstractComparisonReproducer(String expectedErrorMessage) { + this.expectedErrorMessage = expectedErrorMessage; + } + + /** + * Whether the recorded bug has a transformed (second) side. It does not when the bug was a DBMS error triggered by + * the original query alone, in which case there is no second side to evaluate or compare. + * + * @return {@code true} if {@link #evaluateTransformed} should be called + */ + protected abstract boolean hasTransformedSide(); + + /** + * Evaluates the original side against the (reduced) database. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @return the original side's value + * + * @throws SQLException + * if a DBMS interaction fails + */ + protected abstract R evaluateOriginal(G globalState) throws SQLException; + + /** + * Evaluates the transformed side against the (reduced) database. Only called when {@link #hasTransformedSide()} is + * {@code true}. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @return the transformed side's value + * + * @throws SQLException + * if a DBMS interaction fails + */ + protected abstract R evaluateTransformed(G globalState) throws SQLException; + + /** + * Whether the two evaluated sides disagree in the way that constitutes the bug. + * + * @param original + * the original side's value + * @param transformed + * the transformed side's value + * @param globalState + * the state the sides were evaluated against + * + * @return {@code true} if the sides differ (i.e. the bug still triggers) + */ + protected abstract boolean sidesDiffer(R original, R transformed, G globalState); + + @Override + public final boolean bugStillTriggers(G globalState) { + R original; + R transformed; + try { + original = evaluateOriginal(globalState); + if (!hasTransformedSide()) { + // the original bug was a DBMS error on the original query alone, which no longer occurs + return false; + } + transformed = evaluateTransformed(globalState); + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs + return false; + } + return sidesDiffer(original, transformed, globalState); + } + + @Override + public final String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage != null) { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage).append(System.lineSeparator()); + } else { + sb.append(mismatchHeaderLine()).append(System.lineSeparator()); + } + appendQueryLines(sb); + return sb.toString(); + } + + /** + * The header line (without trailing line separator) describing the mismatch, used when the original bug was a + * comparison mismatch rather than an error. For example, "-- On the database set up by the statements above, the + * result sets of the following queries mismatch:". + * + * @return the mismatch header line + */ + protected abstract String mismatchHeaderLine(); + + /** + * Appends the failing queries (or statements) to {@code sb}, one commented line each, so the reduced test case is + * self-contained. Called for both the mismatch and the error case, after the header. + * + * @param sb + * the builder to append to + */ + protected abstract void appendQueryLines(StringBuilder sb); +} diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index b5aedf2cb..c3ad4bae9 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -53,50 +53,38 @@ public class EETOracle, J extends Join, E private Reproducer reproducer; private String generatedQueryString; - private final class EETReproducer implements Reproducer { + private final class EETReproducer extends AbstractComparisonReproducer> { private final String originalQueryString; // null if the original bug was a DBMS error on the original query alone private final String transformedQueryString; - // null if the original bug is a result set mismatch; otherwise, the message of the - // unexpected DBMS error that the original or transformed query triggered - private final String expectedErrorMessage; EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { + super(expectedErrorMessage); this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; - this.expectedErrorMessage = expectedErrorMessage; } @Override - public boolean bugStillTriggers(G globalState) { - List originalResultSet; - List transformedResultSet; - try { - // Re-execute both queries against the current (reduced) database instead of comparing - // against a cached result set, which would be stale once statements have been removed. - originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - globalState); - if (transformedQueryString == null) { - // the original bug was a DBMS error on the original query alone, which no - // longer occurs - return false; - } - transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, - globalState); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (SQLException | RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs - return false; - } + protected boolean hasTransformedSide() { + return transformedQueryString != null; + } + + @Override + protected List evaluateOriginal(G globalState) throws SQLException { + // Re-execute against the current (reduced) database instead of comparing against a cached result set, + // which would be stale once statements have been removed. + return ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + } + + @Override + protected List evaluateTransformed(G globalState) throws SQLException { + return ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); + } + + @Override + protected boolean sidesDiffer(List original, List transformed, G globalState) { try { - ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, + ComparatorHelper.assumeResultSetsAreEqual(original, transformed, originalQueryString, List.of(transformedQueryString), globalState); } catch (AssertionError resultSetMismatch) { return true; @@ -105,21 +93,17 @@ public boolean bugStillTriggers(G globalState) { } @Override - public String getBugInformation() { - StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage == null) { - sb.append("-- On the database set up by the statements above, the result sets of the following" - + " queries mismatch:").append(System.lineSeparator()); - } else { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage) - .append(System.lineSeparator()); - } + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); if (transformedQueryString != null) { sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); } - return sb.toString(); } } diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 35f0c26af..11e52b91d 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -30,63 +30,55 @@ public class NoRECOracle, J extends Join, private Reproducer reproducer; private String lastQueryString; - private static class NoRECReproducer> implements Reproducer { + private static class NoRECReproducer> + extends AbstractComparisonReproducer { private final Function optimizedQuery; private final Function unoptimizedQuery; private final String optimizedQueryString; private final String unoptimizedQueryString; - // null if the original bug is a count mismatch; otherwise, the message of the unexpected - // DBMS error that the original queries triggered - private final String expectedErrorMessage; NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, String optimizedQueryString, String unoptimizedQueryString, String expectedErrorMessage) { + super(expectedErrorMessage); this.optimizedQuery = optimizedQuery; this.unoptimizedQuery = unoptimizedQuery; this.optimizedQueryString = optimizedQueryString; this.unoptimizedQueryString = unoptimizedQueryString; - this.expectedErrorMessage = expectedErrorMessage; } @Override - public boolean bugStillTriggers(G globalState) { - int optimizedCount; - int unoptimizedCount; - try { - optimizedCount = optimizedQuery.apply(globalState); - unoptimizedCount = unoptimizedQuery.apply(globalState); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs - return false; - } + protected boolean hasTransformedSide() { + return true; + } + + @Override + protected Integer evaluateOriginal(G globalState) { + return optimizedQuery.apply(globalState); + } + + @Override + protected Integer evaluateTransformed(G globalState) { + return unoptimizedQuery.apply(globalState); + } + + @Override + protected boolean sidesDiffer(Integer optimizedCount, Integer unoptimizedCount, G globalState) { if (optimizedCount == -1 || unoptimizedCount == -1) { return false; } - return optimizedCount != unoptimizedCount; + return optimizedCount.intValue() != unoptimizedCount.intValue(); } @Override - public String getBugInformation() { - StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage == null) { - sb.append("-- On the database set up by the statements above, the row counts of the following" - + " queries mismatch:").append(System.lineSeparator()); - } else { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage) - .append(System.lineSeparator()); - } + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the row counts of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); - return sb.toString(); } } diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 8fc87ac3d..bf759ebfc 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -29,56 +29,59 @@ public class TLPWhereOracle, J extends Join reproducer; private String generatedQueryString; - private class TLPWhereReproducer implements Reproducer { + // A side's result set, together with the human-readable combined-query strings that + // getCombinedResultSet fills in for the transformed side (unused, and null, for the original side) + private static final class TLPResultSet { + final List resultSet; + final List combinedString; + + TLPResultSet(List resultSet, List combinedString) { + this.resultSet = resultSet; + this.combinedString = combinedString; + } + } + + private class TLPWhereReproducer extends AbstractComparisonReproducer { final String firstQueryString; final String secondQueryString; final String thirdQueryString; final String originalQueryString; final boolean orderBy; - // null if the original bug is a result set mismatch; otherwise, the message of the - // unexpected DBMS error that the original queries triggered - final String expectedErrorMessage; TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, String originalQueryString, boolean orderBy, String expectedErrorMessage) { + super(expectedErrorMessage); this.firstQueryString = firstQueryString; this.secondQueryString = secondQueryString; this.thirdQueryString = thirdQueryString; this.originalQueryString = originalQueryString; this.orderBy = orderBy; - this.expectedErrorMessage = expectedErrorMessage; } @Override - public boolean bugStillTriggers(G globalState) { - List firstResultSet; + protected boolean hasTransformedSide() { + return firstQueryString != null; + } + + @Override + protected TLPResultSet evaluateOriginal(G globalState) throws SQLException { + return new TLPResultSet( + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState), null); + } + + @Override + protected TLPResultSet evaluateTransformed(G globalState) throws SQLException { List combinedString = new ArrayList<>(); - List secondResultSet; - try { - firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - globalState); - if (firstQueryString == null) { - // the original bug was a DBMS error on the original query alone, which no - // longer occurs - return false; - } - secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, globalState, errors); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (SQLException | RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs - return false; - } + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, globalState, errors); + return new TLPResultSet(secondResultSet, combinedString); + } + + @Override + protected boolean sidesDiffer(TLPResultSet original, TLPResultSet transformed, G globalState) { try { - ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, - combinedString, globalState); + ComparatorHelper.assumeResultSetsAreEqual(original.resultSet, transformed.resultSet, + originalQueryString, transformed.combinedString, globalState); } catch (AssertionError resultSetMismatch) { return true; } @@ -86,16 +89,13 @@ public boolean bugStillTriggers(G globalState) { } @Override - public String getBugInformation() { - StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage == null) { - sb.append("-- On the database set up by the statements above, the result sets of the following" - + " queries mismatch:").append(System.lineSeparator()); - } else { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage) - .append(System.lineSeparator()); - } + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { sb.append("-- ").append(originalQueryString).append(';').append(System.lineSeparator()); if (firstQueryString != null) { if (orderBy) { @@ -107,7 +107,6 @@ public String getBugInformation() { .append(" UNION ALL ").append(thirdQueryString).append(';').append(System.lineSeparator()); } } - return sb.toString(); } } From 2a74be3f623f67398c96e31a4e4f75869f18e973 Mon Sep 17 00:00:00 2001 From: splf Date: Thu, 30 Jul 2026 16:08:07 +0500 Subject: [PATCH 114/132] Do not schedule disabled PostgreSQL tablespaces --- src/sqlancer/postgres/PostgresProvider.java | 2 +- .../postgres/TestPostgresProvider.java | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 test/sqlancer/postgres/TestPostgresProvider.java diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index ec7978216..814013df6 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -194,7 +194,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { nrPerformed = r.getInteger(0, 2); break; case CREATE_TABLESPACE: - nrPerformed = r.getInteger(0, 2); + nrPerformed = globalState.getDbmsSpecificOptions().isTestTablespaces() ? r.getInteger(0, 2) : 0; break; case UPDATE: nrPerformed = r.getInteger(0, 10); diff --git a/test/sqlancer/postgres/TestPostgresProvider.java b/test/sqlancer/postgres/TestPostgresProvider.java new file mode 100644 index 000000000..81e795568 --- /dev/null +++ b/test/sqlancer/postgres/TestPostgresProvider.java @@ -0,0 +1,21 @@ +package sqlancer.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class TestPostgresProvider { + + @Test + void createTablespaceIsNotScheduledWhenDisabled() { + PostgresGlobalState state = new PostgresGlobalState(); + state.setDbmsSpecificOptions(new PostgresOptions() { + @Override + public boolean isTestTablespaces() { + return false; + } + }); + + assertEquals(0, PostgresProvider.mapActions(state, PostgresProvider.Action.CREATE_TABLESPACE)); + } +} From e954c10c75c79ac11f4bbf52c2edd00a6d442fdd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 20:22:31 +0800 Subject: [PATCH 115/132] Factor out unexpected-error reproduction into dedicated UnexpectedErrorReproducer --- .../oracle/AbstractComparisonReproducer.java | 73 +++++------------ src/sqlancer/common/oracle/EETOracle.java | 44 +++++++---- src/sqlancer/common/oracle/NoRECOracle.java | 36 ++++++--- .../common/oracle/TLPWhereOracle.java | 60 +++++++++----- .../oracle/UnexpectedErrorReproducer.java | 79 +++++++++++++++++++ 5 files changed, 190 insertions(+), 102 deletions(-) create mode 100644 src/sqlancer/common/oracle/UnexpectedErrorReproducer.java diff --git a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java index d20a9636a..582a19c1d 100644 --- a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java +++ b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java @@ -7,43 +7,28 @@ /** * Shared skeleton for the {@link Reproducer}s of oracles that detect a bug by comparing two evaluations of a - * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}. All of these - * reduce the bug the same way: re-evaluate both sides against the reduced database and report whether they still - * disagree (or, when the original bug was an unexpected DBMS error, whether that same error still fires). + * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}). Reduction re-runs + * both sides against the reduced database and reports whether they still disagree. * *

- * This class owns that control flow (including distinguishing a still-reproducing error from an unrelated one - * introduced by the reduction) and the {@link #getBugInformation()} header. Subclasses supply the parts specific to - * their oracle: how each side is evaluated, how the two are compared, and how the failing queries are rendered in the - * reduced test case. + * The separate case where the original bug was an unexpected DBMS error rather than a mismatch is handled by + * {@link UnexpectedErrorReproducer}, so a subclass here deals only with comparing two sides and never with error + * handling. + * + *

+ * This class owns the compare-and-report control flow and the {@link #getBugInformation()} header. Subclasses supply + * how each side is evaluated, how the two are compared, and how the failing queries are rendered in the reduced test + * case. * * @param * the DBMS-specific global state class * @param - * the type each side evaluates to (e.g. a result set as a list of strings, a row count, a post-image) + * the type each side evaluates to (e.g. a result set as a list of strings, a row count) */ public abstract class AbstractComparisonReproducer, R> implements Reproducer { /** - * The message of the unexpected DBMS error the original bug was, or {@code null} if the original bug was a - * comparison mismatch rather than an error. - */ - protected final String expectedErrorMessage; - - protected AbstractComparisonReproducer(String expectedErrorMessage) { - this.expectedErrorMessage = expectedErrorMessage; - } - - /** - * Whether the recorded bug has a transformed (second) side. It does not when the bug was a DBMS error triggered by - * the original query alone, in which case there is no second side to evaluate or compare. - * - * @return {@code true} if {@link #evaluateTransformed} should be called - */ - protected abstract boolean hasTransformedSide(); - - /** - * Evaluates the original side against the (reduced) database. + * Evaluates the original side against the reduced database. * * @param globalState * the state whose connection points at the reduced database @@ -56,8 +41,7 @@ protected AbstractComparisonReproducer(String expectedErrorMessage) { protected abstract R evaluateOriginal(G globalState) throws SQLException; /** - * Evaluates the transformed side against the (reduced) database. Only called when {@link #hasTransformedSide()} is - * {@code true}. + * Evaluates the transformed side against the reduced database. * * @param globalState * the state whose connection points at the reduced database @@ -89,21 +73,9 @@ public final boolean bugStillTriggers(G globalState) { R transformed; try { original = evaluateOriginal(globalState); - if (!hasTransformedSide()) { - // the original bug was a DBMS error on the original query alone, which no longer occurs - return false; - } transformed = evaluateTransformed(globalState); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (SQLException | RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs + } catch (AssertionError | SQLException | RuntimeException e) { + // any failure re-running the two sides means this reduced database no longer shows the mismatch return false; } return sidesDiffer(original, transformed, globalState); @@ -112,28 +84,21 @@ public final boolean bugStillTriggers(G globalState) { @Override public final String getBugInformation() { StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage != null) { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage).append(System.lineSeparator()); - } else { - sb.append(mismatchHeaderLine()).append(System.lineSeparator()); - } + sb.append(mismatchHeaderLine()).append(System.lineSeparator()); appendQueryLines(sb); return sb.toString(); } /** - * The header line (without trailing line separator) describing the mismatch, used when the original bug was a - * comparison mismatch rather than an error. For example, "-- On the database set up by the statements above, the - * result sets of the following queries mismatch:". + * The header line (without trailing line separator) describing the mismatch. For example, "-- On the database set + * up by the statements above, the result sets of the following queries mismatch:". * * @return the mismatch header line */ protected abstract String mismatchHeaderLine(); /** - * Appends the failing queries (or statements) to {@code sb}, one commented line each, so the reduced test case is - * self-contained. Called for both the mismatch and the error case, after the header. + * Appends the failing queries to {@code sb}, one commented line each, so the reduced test case is self-contained. * * @param sb * the builder to append to diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index c3ad4bae9..0cd4ae99e 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -55,20 +55,13 @@ public class EETOracle, J extends Join, E private final class EETReproducer extends AbstractComparisonReproducer> { private final String originalQueryString; - // null if the original bug was a DBMS error on the original query alone private final String transformedQueryString; - EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { - super(expectedErrorMessage); + EETReproducer(String originalQueryString, String transformedQueryString) { this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; } - @Override - protected boolean hasTransformedSide() { - return transformedQueryString != null; - } - @Override protected List evaluateOriginal(G globalState) throws SQLException { // Re-execute against the current (reduced) database instead of comparing against a cached result set, @@ -100,11 +93,32 @@ protected String mismatchHeaderLine() { @Override protected void appendQueryLines(StringBuilder sb) { - sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + renderQueryLines(sb, originalQueryString, transformedQueryString); + } + } + + // Renders the failing queries as commented lines, shared by the mismatch and the unexpected-error reproducers. + // transformedQueryString is null when the error struck the original query before any transformation existed. + private static void renderQueryLines(StringBuilder sb, String originalQueryString, String transformedQueryString) { + sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (transformedQueryString != null) { + sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); + } + } + + // Builds the reproducer for an unexpected DBMS error, which re-runs the query (or both queries) and checks the same + // error still fires. transformedQueryString is null when only the original query ran before the error. + private UnexpectedErrorReproducer errorReproducer(String originalQueryString, String transformedQueryString, + String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); if (transformedQueryString != null) { - sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); + ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); } - } + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, originalQueryString, transformedQueryString); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); } public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { @@ -139,8 +153,8 @@ public void check() throws SQLException { originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); } catch (AssertionError unexpectedError) { // an unexpected DBMS error on the original query alone is itself a bug worth reducing; - // transformedQueryString is null because no transformed query is involved - reproducer = new EETReproducer(originalQueryString, null, + // there is no transformed query yet, so only the original is replayed + reproducer = errorReproducer(originalQueryString, null, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } @@ -160,14 +174,14 @@ public void check() throws SQLException { } catch (AssertionError unexpectedError) { // the semantics-preserving transformation made the query trigger a DBMS error that the // original did not, which is a bug worth reducing - reproducer = new EETReproducer(originalQueryString, transformedQueryString, + reproducer = errorReproducer(originalQueryString, transformedQueryString, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } // Set the reproducer before the assertion: assumeResultSetsAreEqual throws when the bug is // detected, so creating the reproducer afterwards would leave it null and prevent any reduction. - reproducer = new EETReproducer(originalQueryString, transformedQueryString, null); + reproducer = new EETReproducer(originalQueryString, transformedQueryString); ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), state); diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 11e52b91d..ba48e80bb 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -38,19 +38,13 @@ private static class NoRECReproducer> private final String unoptimizedQueryString; NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, - String optimizedQueryString, String unoptimizedQueryString, String expectedErrorMessage) { - super(expectedErrorMessage); + String optimizedQueryString, String unoptimizedQueryString) { this.optimizedQuery = optimizedQuery; this.unoptimizedQuery = unoptimizedQuery; this.optimizedQueryString = optimizedQueryString; this.unoptimizedQueryString = unoptimizedQueryString; } - @Override - protected boolean hasTransformedSide() { - return true; - } - @Override protected Integer evaluateOriginal(G globalState) { return optimizedQuery.apply(globalState); @@ -77,11 +71,29 @@ protected String mismatchHeaderLine() { @Override protected void appendQueryLines(StringBuilder sb) { - sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); - sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + renderQueryLines(sb, optimizedQueryString, unoptimizedQueryString); } } + // Renders the failing queries as commented lines, shared by the mismatch and the unexpected-error reproducers. + private static void renderQueryLines(StringBuilder sb, String optimizedQueryString, String unoptimizedQueryString) { + sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); + sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + } + + // Builds the reproducer for an unexpected DBMS error, which re-runs both queries and checks the same error fires. + private static > UnexpectedErrorReproducer errorReproducer( + Function optimizedQuery, Function unoptimizedQuery, String optimizedQueryString, + String unoptimizedQueryString, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + optimizedQuery.apply(globalState); + unoptimizedQuery.apply(globalState); + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, optimizedQueryString, unoptimizedQueryString); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -128,8 +140,8 @@ public void check() throws SQLException { optimizedCount = optimizedQuery.apply(state); unoptimizedCount = unoptimizedQuery.apply(state); } catch (AssertionError unexpectedError) { - reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, - unoptimizedQueryString, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + reproducer = errorReproducer(optimizedQuery, unoptimizedQuery, optimizedQueryString, unoptimizedQueryString, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } @@ -139,7 +151,7 @@ public void check() throws SQLException { if (unoptimizedCount != optimizedCount) { reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, - unoptimizedQueryString, null); + unoptimizedQueryString); String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index bf759ebfc..5e1d2861b 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -49,8 +49,7 @@ private class TLPWhereReproducer extends AbstractComparisonReproducer errorReproducer(String originalQueryString, String firstQueryString, + String secondQueryString, String thirdQueryString, boolean orderBy, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + if (firstQueryString != null) { + ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, + new ArrayList<>(), !orderBy, globalState, errors); + } + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, originalQueryString, firstQueryString, secondQueryString, thirdQueryString, orderBy); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -140,7 +158,7 @@ public void check() throws SQLException { try { firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); } catch (AssertionError unexpectedError) { - reproducer = new TLPWhereReproducer(null, null, null, originalQueryString, false, + reproducer = errorReproducer(originalQueryString, null, null, null, false, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } @@ -165,13 +183,13 @@ public void check() throws SQLException { secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); } catch (AssertionError unexpectedError) { - reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, - originalQueryString, orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + reproducer = errorReproducer(originalQueryString, firstQueryString, secondQueryString, thirdQueryString, + orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - orderBy, null); + orderBy); ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); } diff --git a/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java b/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java new file mode 100644 index 000000000..e9390943a --- /dev/null +++ b/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java @@ -0,0 +1,79 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; + +/** + * Reproducer for a bug that is an unexpected DBMS error, rather than a violation of oracle logic. When a statement run + * by the oracle raises an error the oracle did not expect, the bug is that error. Reduction re-runs the statement + * execution against the reduced database and reports whether the same error still fires. The oracle supplies how to + * re-run its execution as an {@link Execution} functional interface. + * + *

+ * For an oracle to use this reproducer on its unexpected errors, they must surface as {@link AssertionError}s, even + * though they likely originated as {@link SQLException}s. This is because + * {@link UnexpectedErrorReproducer#bugStillTriggers} treats a {@link SQLException} as a replay failure that means "bug + * does not trigger" (e.g. dropped connection, removed table). + * + * @param + * the DBMS-specific global state class + */ +public final class UnexpectedErrorReproducer> implements Reproducer { + + private final Execution execution; + private final String expectedErrorMessage; + private final String queryLines; + + @FunctionalInterface + public interface Execution { + /** + * Re-runs the oracle's execution against the reduced database. The bug is treated as still present if the run + * continues to raise an {@link AssertionError} with the same message. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @throws SQLException + * if a DBMS interaction fails for a reason other than the recorded bug (e.g. a connection or setup + * failure during replay), which counts as the bug no longer triggering + */ + void execute(G globalState) throws SQLException; + } + + /** + * @param execution + * re-runs the oracle's execution against the reduced database + * @param expectedErrorMessage + * the message of the error the original bug was. The bug is treated as still present if the + * {@link Execution} continues to raise an {@link AssertionError} with the same message. + * @param queryLines + * the failing queries as commented lines (each ending in a line separator), for the reduced test case + */ + public UnexpectedErrorReproducer(Execution execution, String expectedErrorMessage, String queryLines) { + this.execution = execution; + this.expectedErrorMessage = expectedErrorMessage; + this.queryLines = queryLines; + } + + @Override + public boolean bugStillTriggers(G globalState) { + try { + execution.execute(globalState); + } catch (AssertionError unexpectedError) { + // the same error reproduces the bug; a different one is an artifact of the reduction (e.g. a removed table) + return expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + // the error no longer fires + return false; + } + + @Override + public String getBugInformation() { + return "-- On the database set up by the statements above, the following queries trigger an unexpected error" + + " with message: " + expectedErrorMessage + System.lineSeparator() + queryLines; + } +} From 96ab07d6fd1abde8a6f53bfa59b0f6736fbff79b Mon Sep 17 00:00:00 2001 From: splf Date: Fri, 31 Jul 2026 10:07:42 +0500 Subject: [PATCH 116/132] Expect the to_char error for RN combined with other formats PostgreSQL rejects a numeric format string that combines RN with any other pattern, for example to_char(1.5, 'RN0'), with the error "RN" is incompatible with other formats. The expression generator can produce such format strings, so this error reached the caller as an unexpected one and aborted the test run. --- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index eeb160a56..180b4e449 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -123,6 +123,7 @@ private static List getToCharFunctionErrors() { errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); errors.add("\"EEEE\" must be the last pattern used"); + errors.add("is incompatible with other formats"); return errors; } From adc0875fe76c4d48939fc4be2a0097ee715e521a Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 28 Jul 2026 16:40:37 +0800 Subject: [PATCH 117/132] Add support for DECIMAL(M, D) columns in MySQL EET --- src/sqlancer/mysql/MySQLSchema.java | 12 ++- .../mysql/ast/MySQLCastOperation.java | 80 +++++++++++++++++-- .../mysql/gen/MySQLTableGenerator.java | 21 +++-- .../mysql/oracle/MySQLEETTransformer.java | 27 ++++--- .../mysql/MySQLToStringVisitorTest.java | 6 +- .../mysql/ast/MySQLCaseOperatorTest.java | 4 +- 6 files changed, 120 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index c8a30614f..ab0ba4543 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -57,6 +57,7 @@ public static class MySQLColumn extends AbstractTableColumn getTableColumns(SQLConnection con, String table String columnName = rs.getString("COLUMN_NAME"); String dataType = rs.getString("DATA_TYPE"); int precision = rs.getInt("NUMERIC_PRECISION"); + int scale = rs.getInt("NUMERIC_SCALE"); boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); - MySQLColumn c = new MySQLColumn(columnName, getColumnType(dataType), isPrimaryKey, precision); + MySQLColumn c = new MySQLColumn(columnName, getColumnType(dataType), isPrimaryKey, precision, + scale); columns.add(c); } } diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index b71d0498c..07c45a429 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -1,21 +1,91 @@ package sqlancer.mysql.ast; +import java.util.Objects; + public class MySQLCastOperation implements MySQLExpression { private final MySQLExpression expr; private final CastType type; - public enum CastType { - SIGNED, UNSIGNED, - // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never - // evaluated, so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). - CHAR, FLOAT, DOUBLE, DECIMAL; + /** + * A MySQL {@code CAST} target type. The non-{@code DECIMAL} kinds are interned singletons; {@code DECIMAL} may + * additionally carry an {@code (M, D)} precision/scale (via {@link #decimal}) so that a + * {@code CAST(... AS DECIMAL(M, D))} can reproduce a column's exact type. This is relied on by the EET oracle's + * type-pinning casts (see {@code MySQLEETTransformer}). + */ + public static final class CastType { + + // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never evaluated, + // so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + public static final CastType SIGNED = new CastType(Kind.SIGNED); + public static final CastType UNSIGNED = new CastType(Kind.UNSIGNED); + public static final CastType CHAR = new CastType(Kind.CHAR); + public static final CastType FLOAT = new CastType(Kind.FLOAT); + public static final CastType DOUBLE = new CastType(Kind.DOUBLE); + public static final CastType DECIMAL = new CastType(Kind.DECIMAL); + + private enum Kind { + SIGNED, UNSIGNED, CHAR, FLOAT, DOUBLE, DECIMAL + } + + private final Kind kind; + private final Integer precision; // DECIMAL only, otherwise null + private final Integer scale; // DECIMAL only, otherwise null + + private CastType(Kind kind) { + this(kind, null, null); + } + + private CastType(Kind kind, Integer precision, Integer scale) { + this.kind = kind; + this.precision = precision; + this.scale = scale; + } + + // A DECIMAL(precision, scale) cast target. + public static CastType decimal(int precision, int scale) { + return new CastType(Kind.DECIMAL, precision, scale); + } public static CastType getRandom() { return SIGNED; // return Randomly.fromOptions(CastType.SIGNED, CastType.UNSIGNED); } + public Integer getPrecision() { + return precision; + } + + public Integer getScale() { + return scale; + } + + @Override + public String toString() { + if (precision == null) { + return kind.name(); + } + return kind.name() + "(" + precision + ", " + scale + ")"; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof CastType)) { + return false; + } + CastType other = (CastType) obj; + return kind == other.kind && Objects.equals(precision, other.precision) + && Objects.equals(scale, other.scale); + } + + @Override + public int hashCode() { + return Objects.hash(kind, precision, scale); + } + } public MySQLCastOperation(MySQLExpression expr, CastType type) { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 40e325041..b7c9a3566 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -347,11 +347,11 @@ private void appendType(MySQLDataType randomType) { break; case FLOAT: sb.append("FLOAT"); - optionallyAddPrecisionAndScale(sb); + optionallyAddFloatingPointPrecisionAndScale(sb); break; case DOUBLE: sb.append(Randomly.fromOptions("DOUBLE", "FLOAT")); - optionallyAddPrecisionAndScale(sb); + optionallyAddFloatingPointPrecisionAndScale(sb); break; default: throw new AssertionError(); @@ -368,13 +368,20 @@ private void appendType(MySQLDataType randomType) { } } - private void optionallyAddPrecisionAndScale(StringBuilder sb) { - // The EET oracle's type inference assumes FLOAT/DOUBLE/DECIMAL columns are created without (M, D) (see - // MySQLEETTransformer#inferColumnType), so precision/scale is omitted while EET is active. This restriction can - // be lifted once (M, D) is tracked through the codebase and reflected in the CAST target types. + // FLOAT(M, D)/DOUBLE(M, D) is deprecated and cannot be reproduced as a CAST target, so the EET oracle's type + // inference relies on FLOAT/DOUBLE columns being created without (M, D) (see MySQLEETTransformer#inferColumnType); + // it is therefore omitted while EET is active. DECIMAL(M, D) has no such restriction: the EET oracle tracks its + // (M, D) and reproduces it via CAST(... AS DECIMAL(M, D)), so it keeps using optionallyAddPrecisionAndScale. + private void optionallyAddFloatingPointPrecisionAndScale(StringBuilder sb) { boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.EET); - if (Randomly.getBoolean() && !MySQLBugs.bug99183 && !eetActive) { + if (!eetActive) { + optionallyAddPrecisionAndScale(sb); + } + } + + private void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean() && !MySQLBugs.bug99183) { sb.append("("); // The maximum number of digits (M) for DECIMAL is 65 long m = Randomly.getNotCachedInteger(1, 65); diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 2e4394eec..22c0302df 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -5,6 +5,7 @@ import java.util.stream.Collectors; import sqlancer.common.oracle.EETTransformer; +import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; @@ -33,11 +34,11 @@ * *

* MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target - * types ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning - * {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping - * it in a CAST. + * types ({@link CastType}, which carries {@code (M, D)} for DECIMAL): {@link #inferType} conservatively classifies AST + * nodes into that domain (returning {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type + * of a random expression by wrapping it in a CAST. */ -public class MySQLEETTransformer extends EETTransformer { +public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; @@ -212,20 +213,22 @@ private CastType inferConstantType(MySQLConstant constant) { } private CastType inferColumnType(MySQLColumnReference ref) { - switch (ref.getColumn().getType()) { + MySQLColumn column = ref.getColumn(); + switch (column.getType()) { case INT: return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns case VARCHAR: return CastType.CHAR; - // FLOAT/DOUBLE/DECIMAL columns are created without (M, D) while EET is active, so the plain - // CAST target below matches the column's type. Reintroducing (M, D) for better coverage would - // require tracking it here and emitting the exact precision/scale in the CAST. case FLOAT: + // FLOAT/DOUBLE columns are created without (M, D) while EET is active (the (M, D) form is deprecated and + // not a valid CAST target), so the plain CAST target matches the column's type. return CastType.FLOAT; case DOUBLE: return CastType.DOUBLE; case DECIMAL: - return CastType.DECIMAL; + // DECIMAL columns may carry (M, D); CAST(... AS DECIMAL(M, D)) reproduces the column's exact type. The + // schema reports (M, D) even for a plain DECIMAL column (defaulting to (10, 0)). + return CastType.decimal(column.getPrecision(), column.getScale()); default: return null; } @@ -259,7 +262,8 @@ private CastType inferCaseType(MySQLCaseOperator caseOp) { /** * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same - * inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * inferrable type (a conservative under-approximation of MySQL's aggregation rules). Two DECIMAL subexpressions + * with differing {@code (M, D)} therefore yield {@code null} rather than a guessed aggregate. * * @param exprs * the result-type-determining subexpressions @@ -270,7 +274,8 @@ private CastType commonType(MySQLExpression... exprs) { CastType common = null; for (MySQLExpression expr : exprs) { CastType type = inferType(expr); - if (type == null || common != null && type != common) { + // equals (not ==) so two DECIMAL types with matching (M, D) but distinct instances compare as equal. + if (type == null || common != null && !type.equals(common)) { return null; } common = type; diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 3f991e3f1..d39c2695c 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -17,7 +17,7 @@ public class MySQLToStringVisitorTest { @Test void visitAggregateToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCount = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT); @@ -35,7 +35,7 @@ void visitAggregateToString() { @Test void visitAggregateWithDistinctToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCountDistinct = new MySQLAggregate(List.of(aRef), @@ -57,7 +57,7 @@ void visitAggregateWithDistinctToString() { @Test void visitCaseWhenToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); diff --git a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java index 674ab027e..757241445 100644 --- a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java +++ b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java @@ -14,7 +14,7 @@ public class MySQLCaseOperatorTest { @Test void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createIntConstant(1)); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); @@ -29,7 +29,7 @@ void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { @Test void getExpectedValue_switchConditionHasNoMatches_ReturnsElse() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createNullConstant()); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); From 9b739c0501f848c0a0907728356bbe43bed926c4 Mon Sep 17 00:00:00 2001 From: splf Date: Mon, 3 Aug 2026 12:49:45 +0500 Subject: [PATCH 118/132] Do not return null when PostgreSQL tablespaces are disabled --- .../gen/PostgresTableSpaceGenerator.java | 7 ++++-- .../gen/TestPostgresTableSpaceGenerator.java | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java index 3890d5160..1be99ca61 100644 --- a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -1,5 +1,6 @@ package sqlancer.postgres.gen; +import sqlancer.IgnoreMeException; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -19,10 +20,12 @@ public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { } public static SQLQueryAdapter generate(PostgresGlobalState globalState) { - // Skip tablespace generation if the option is disabled + // PostgresProvider.mapActions does not schedule this action when the option is disabled, but QPG + // selects actions by index without consulting the schedule, so the generator has to report that + // it has nothing to generate. PostgresOptions options = globalState.getDbmsSpecificOptions(); if (!options.isTestTablespaces()) { - return null; + throw new IgnoreMeException(); } return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); } diff --git a/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java b/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java new file mode 100644 index 000000000..c2f626041 --- /dev/null +++ b/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.postgres.gen; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import sqlancer.IgnoreMeException; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; + +class TestPostgresTableSpaceGenerator { + + @Test + void generateIsSkippedWhenTablespacesAreDisabled() { + PostgresGlobalState state = new PostgresGlobalState(); + state.setDbmsSpecificOptions(new PostgresOptions() { + @Override + public boolean isTestTablespaces() { + return false; + } + }); + + assertThrows(IgnoreMeException.class, () -> PostgresTableSpaceGenerator.generate(state)); + } +} From 6ac84559193fa5f004ac6c18a020ade4a86ba68b Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 13:37:48 +0800 Subject: [PATCH 119/132] Add support for DELETE statements in EET (with MySQL-specific implementation), including generic DML infrastructure for future extensions to other DML statements --- src/sqlancer/common/gen/EETDMLGenerator.java | 155 +++++++++++++++++ src/sqlancer/common/oracle/EETDMLOracle.java | 158 ++++++++++++++++++ src/sqlancer/mysql/MySQLErrors.java | 17 ++ src/sqlancer/mysql/MySQLGlobalState.java | 9 + src/sqlancer/mysql/MySQLOracleFactory.java | 10 ++ .../mysql/gen/MySQLExpressionGenerator.java | 22 ++- .../mysql/gen/MySQLTableGenerator.java | 14 +- 7 files changed, 378 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/common/gen/EETDMLGenerator.java create mode 100644 src/sqlancer/common/oracle/EETDMLOracle.java diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java new file mode 100644 index 000000000..293574a36 --- /dev/null +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -0,0 +1,155 @@ +package sqlancer.common.gen; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.oracle.EETTransformer; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * Generator interface used by {@link sqlancer.common.oracle.EETDMLOracle}, the DML counterpart of {@link EETGenerator}. + * It supplies methods which generate the transformable expressions, create the DBMS-specific {@link EETTransformer} + * that rewrites them, and produce the SQL of the statements the oracle uses to observe the database state a statement + * produces (an approach drawn from the DQE oracle). + * + *

+ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which + * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run + * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are + * produced. + * + *

+ * Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ +public interface EETDMLGenerator, T extends AbstractTable, C extends AbstractTableColumn> { + + /** Name of the auxiliary column that uniquely identifies each row. */ + String ROW_ID_COLUMN = "rowid"; + + /** + * Restricts this generator to the given tables (a single table, for the DML statement under test) and their + * columns. + * + * @param tables + * the tables (and, implicitly, columns) the generated statement operates on + * + * @return this generator + */ + EETDMLGenerator setTablesAndColumns(AbstractTables tables); + + /** + * Generates a fresh random boolean expression over the current tables' columns, used as the DML statement's WHERE + * predicate. + * + * @return a fresh random boolean expression + */ + E generateBooleanExpression(); + + /** + * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's + * expressions into semantically equivalent ones. + * + * @return a DBMS-specific {@link EETTransformer} + */ + EETTransformer createTransformer(); + + // --- DBMS-specific primitives --- + + /** + * Renders an expression to its DBMS-specific SQL string. + * + * @param expr + * the expression to render + * + * @return the SQL text of {@code expr} + */ + String asString(E expr); + + /** + * SQL that assigns every existing row of {@code table} a distinct, stable identifier in the {@link #ROW_ID_COLUMN} + * column. DBMS-specific because it names the DBMS's UUID-generating function. + * + * @param table + * the table whose rows are stamped + * + * @return the SQL statement + */ + String stampRowIdsStatement(T table); + + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- + + /** + * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}. + * + * @param table + * the table to add the column to + * + * @return the SQL statement + */ + default String addRowIdColumnStatement(T table) { + return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " VARCHAR(36)"; + } + + /** + * SQL that drops the auxiliary {@link #ROW_ID_COLUMN} column from {@code table}. + * + * @param table + * the table to drop the column from + * + * @return the SQL statement + */ + default String dropRowIdColumnStatement(T table) { + return "ALTER TABLE " + table.getName() + " DROP COLUMN " + ROW_ID_COLUMN; + } + + /** + * SQL that selects the {@link #ROW_ID_COLUMN} of every row of {@code table} (the surviving-row snapshot). + * + * @param table + * the table to snapshot + * + * @return the SQL statement; its first result column must be the identifiers + */ + default String selectRowIdsStatement(T table) { + return "SELECT " + ROW_ID_COLUMN + " FROM " + table.getName(); + } + + /** + * SQL that deletes the rows of {@code table} matching {@code predicate}. + * + * @param table + * the table to delete from + * @param predicate + * the WHERE predicate; rendered via {@link #asString} + * + * @return the SQL statement + */ + default String deleteStatement(T table, E predicate) { + return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); + } + + /** + * SQL that starts a transaction, so a statement's effect can be observed and then undone. + * + * @return the SQL statement + */ + default String beginTransactionStatement() { + return "BEGIN"; + } + + /** + * SQL that rolls the current transaction back, undoing the statement's effect. + * + * @return the SQL statement + */ + default String rollbackTransactionStatement() { + return "ROLLBACK"; + } +} diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java new file mode 100644 index 000000000..33f9ff411 --- /dev/null +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -0,0 +1,158 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.gen.EETDMLGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * EET (Equivalent Expression Transformation) oracle for DML statements, based on "Detecting Logic Bugs in Database + * Engines via Equivalent Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * Whereas {@link EETOracle} transforms a SELECT and compares the two result sets, this oracle transforms a DML + * statement and compares the two database states produced. + * + *

+ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which + * uniquely identifies each row, and each statement is executed inside a transaction that is rolled back, so the two + * statements can be compared against the same starting state without permanently modifying the database. For a DELETE, + * the state is captured as the set of surviving row identifiers. Because rolling back a statement requires a + * transactional storage engine, the DBMS-specific setup must ensure only such engines are used while this oracle is + * active. + * + *

+ * Only DELETE is currently supported. Statement reduction is not yet implemented (there is no + * {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database reduction. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific schema class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + * @param + * the DBMS-specific global state class + */ +public class EETDMLOracle, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private EETDMLGenerator gen; + private final EETTransformer transformer; + private final ExpectedErrors errors; + + private String generatedQueryString; + + public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.transformer = gen.createTransformer(); + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + List tables = state.getSchema().getDatabaseTables(); + if (tables.isEmpty()) { + throw new IgnoreMeException(); + } + // DELETE targets a single table, so operate on exactly one; confining the generator to it keeps the predicate + // from referencing another table's columns (which would render invalid single-table DML). + T table = Randomly.fromList(tables); + gen = gen.setTablesAndColumns(new AbstractTables<>(List.of(table))); + + E predicate = gen.generateBooleanExpression(); + // The WHERE predicate is evaluated in a boolean context. + E transformedPredicate = transformer.transform(predicate, true); + + String originalDelete = gen.deleteStatement(table, predicate); + String transformedDelete = gen.deleteStatement(table, transformedPredicate); + generatedQueryString = originalDelete; + + // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: + // the ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak + // the column into the next iteration and cause cascading duplicate-column failures. + new SQLQueryAdapter(gen.addRowIdColumnStatement(table), true).execute(state); + try { + // Stamp identifiers once, in autocommit mode, before both DELETEs run: both then observe the same rows. + new SQLQueryAdapter(gen.stampRowIdsStatement(table)).execute(state); + + Set originalSurvivors = executeDeleteAndSnapshot(table, originalDelete); + Set transformedSurvivors = executeDeleteAndSnapshot(table, transformedDelete); + + if (!originalSurvivors.equals(transformedSurvivors)) { + throw new AssertionError( + mismatchMessage(originalDelete, transformedDelete, originalSurvivors, transformedSurvivors)); + } + } finally { + new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), true).execute(state); + } + } + + /** + * Executes {@code deleteStatement} inside a transaction that is always rolled back, and returns the set of row + * identifiers surviving the DELETE (the resulting database state). A DBMS error expected by the oracle aborts the + * whole check ({@link IgnoreMeException}) rather than being reported, matching {@link EETOracle}'s handling; an + * unexpected error surfaces as a bug ({@link AssertionError}, thrown by the query adapter). + * + * @param table + * the table being deleted from + * @param deleteStatement + * the DELETE statement to execute + * + * @return the set of row identifiers surviving the DELETE + * + * @throws SQLException + * if a DBMS interaction fails + */ + private Set executeDeleteAndSnapshot(T table, String deleteStatement) throws SQLException { + new SQLQueryAdapter(gen.beginTransactionStatement()).execute(state); + try { + // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. + boolean succeeded = new SQLQueryAdapter(deleteStatement, errors).execute(state); + if (!succeeded) { + // The DELETE hit an error the oracle tolerates; do not compare states (as EETOracle does for SELECT). + throw new IgnoreMeException(); + } + return new HashSet<>( + ComparatorHelper.getResultSetFirstColumnAsString(gen.selectRowIdsStatement(table), errors, state)); + } finally { + new SQLQueryAdapter(gen.rollbackTransactionStatement()).execute(state); + } + } + + private static String mismatchMessage(String originalDelete, String transformedDelete, + Set originalSurvivors, Set transformedSurvivors) { + return new StringBuilder() + .append("-- The original and transformed DELETE statements left the database in different states") + .append(" (different sets of surviving rows):").append(System.lineSeparator()).append("-- original (") + .append(originalSurvivors.size()).append(" rows survive): ").append(originalDelete).append(';') + .append(System.lineSeparator()).append("-- transformed (").append(transformedSurvivors.size()) + .append(" rows survive): ").append(transformedDelete).append(';').append(System.lineSeparator()) + .toString(); + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index cb9ad4f01..44316dc3f 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -65,4 +65,21 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { errors.addAll(getInsertUpdateErrors()); } + public static List getDMLErrors() { + ArrayList errors = new ArrayList<>(); + + // WHERE-clause type coercion (e.g. string -> number) is only a warning in SELECT but a hard error in + // DELETE/UPDATE under strict sql_mode (MySQL 1292). A semantics-preserving transform may benignly change + // whether it fires, so it is tolerated rather than flagged. + errors.add("Truncated incorrect"); + // Foreign key constraint failure when deleting/updating a referenced row. + errors.add("a foreign key constraint fails"); + + return errors; + } + + public static void addDMLErrors(ExpectedErrors errors) { + errors.addAll(getDMLErrors()); + } + } diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 10132b57c..a34861cbd 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -16,4 +16,13 @@ public boolean usesPQS() { return getDbmsSpecificOptions().oracles.stream().anyMatch(o -> o == MySQLOracleFactory.PQS); } + public boolean usesEET() { + return getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.EET || o == MySQLOracleFactory.EET_DML); + } + + public boolean usesEETDML() { + return getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.EET_DML); + } + } diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index ed5ddc489..d3d4b20b1 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -5,6 +5,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.EETDMLOracle; import sqlancer.common.oracle.EETOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; @@ -92,5 +93,14 @@ public TestOracle create(MySQLGlobalState globalState) throws .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); return new EETOracle<>(globalState, gen, expectedErrors); } + }, + EET_DML { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).with(MySQLErrors.getDMLErrors()).build(); + return new EETDMLOracle<>(globalState, gen, expectedErrors); + } }; } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index da304ac67..5a694bd78 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.EETDMLGenerator; import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; @@ -19,6 +20,7 @@ import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLBetweenOperation; @@ -52,7 +54,8 @@ public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, CERTGenerator, - EETGenerator { + EETGenerator, + EETDMLGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; @@ -364,10 +367,25 @@ boolean mutateOr(MySQLSelect select) { } } - // --- EET oracle --- + // --- EET oracle (including DML) --- @Override public EETTransformer createTransformer() { return new MySQLEETTransformer(this); } + + // --- EET DML only --- + + @Override + public String asString(MySQLExpression expr) { + return MySQLVisitor.asString(expr); + } + + @Override + public String stampRowIdsStatement(MySQLTable table) { + // MySQL's UUID() gives each existing row a distinct value in a single statement. Stamping happens once, before + // both rolled-back DELETE runs, so both observe identical identifiers; the standard-SQL statements (add/drop + // column, delete, snapshot, transaction control) use EETDMLGenerator's defaults. + return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); + } } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index b7c9a3566..0746056ad 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -198,7 +198,10 @@ private void appendTableOptions() { // "NDB": java.sql.SQLSyntaxErrorException: Unknown storage engine 'NDB' // "EXAMPLE": java.sql.SQLSyntaxErrorException: Unknown storage engine 'EXAMPLE' // "MERGE": java.sql.SQLException: Table 't0' is read only - String fromOptions = Randomly.fromOptions("InnoDB", "MyISAM", "MEMORY", "HEAP", "CSV", "ARCHIVE"); + // The EET DML oracle rolls back each statement to compare database states, which requires a + // transactional engine, so only InnoDB is used while it is active. + String fromOptions = globalState.usesEETDML() ? "InnoDB" + : Randomly.fromOptions("InnoDB", "MyISAM", "MEMORY", "HEAP", "CSV", "ARCHIVE"); this.engine = MySQLEngine.get(fromOptions); sb.append("ENGINE = "); sb.append(fromOptions); @@ -362,7 +365,8 @@ private void appendType(MySQLDataType randomType) { } if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS - || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET)) { + || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET + || o == MySQLOracleFactory.EET_DML)) { sb.append(" ZEROFILL"); } } @@ -373,9 +377,9 @@ private void appendType(MySQLDataType randomType) { // it is therefore omitted while EET is active. DECIMAL(M, D) has no such restriction: the EET oracle tracks its // (M, D) and reproduces it via CAST(... AS DECIMAL(M, D)), so it keeps using optionallyAddPrecisionAndScale. private void optionallyAddFloatingPointPrecisionAndScale(StringBuilder sb) { - boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() - .anyMatch(o -> o == MySQLOracleFactory.EET); - if (!eetActive) { + // Both EET oracles rely on the same type inference (MySQLEETTransformer), so both omit FLOAT(M, D)/DOUBLE(M, + // D). + if (!globalState.usesEET()) { optionallyAddPrecisionAndScale(sb); } } From 514f14959e295f56981514248bf8a96660596ccd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 15:10:55 +0800 Subject: [PATCH 120/132] Add support for LIMIT on DELETE statements in EET --- src/sqlancer/common/gen/EETDMLGenerator.java | 32 ++++++++++++++++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 13 ++++++-- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 293574a36..375877a63 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -1,5 +1,8 @@ package sqlancer.common.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.ast.newast.Expression; import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTable; @@ -122,17 +125,40 @@ default String selectRowIdsStatement(T table) { } /** - * SQL that deletes the rows of {@code table} matching {@code predicate}. + * SQL that deletes the rows of {@code table} matching {@code predicate}, optionally limited to the first + * {@code limit} rows. + * + *

+ * When {@code limit} is non-null, the statement is ordered by {@code orderByColumns} followed by + * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even when + * the ordering columns tie), so the "first {@code limit}" rows are identical for the original and + * transformed statements. Varying the ordering columns exercises more access + * paths than the row id alone would. The caller must pass the same {@code orderByColumns} and {@code limit} + * to both statements; neither is transformed. * * @param table * the table to delete from * @param predicate * the WHERE predicate; rendered via {@link #asString} + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty); only used when {@code limit} is + * non-null + * @param limit + * the maximum number of rows to delete, or {@code null} for no limit * * @return the SQL statement */ - default String deleteStatement(T table, E predicate) { - return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); + default String deleteStatement(T table, E predicate, List orderByColumns, Integer limit) { + String statement = "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); + if (limit != null) { + List orderBy = new ArrayList<>(); + for (C column : orderByColumns) { + orderBy.add(column.getName()); + } + orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above + statement += " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; + } + return statement; } /** diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 33f9ff411..792bc52c0 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -84,8 +84,17 @@ public void check() throws SQLException { // The WHERE predicate is evaluated in a boolean context. E transformedPredicate = transformer.transform(predicate, true); - String originalDelete = gen.deleteStatement(table, predicate); - String transformedDelete = gen.deleteStatement(table, transformedPredicate); + // Optionally cap the DELETE with a LIMIT. The limit and its ordering (a random column subset, made a total + // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped + // row set is deterministic and equal across the runs while still exercising varied orderings. + Integer limit = null; + List orderByColumns = List.of(); + if (Randomly.getBoolean()) { + limit = (int) Randomly.getNotCachedInteger(0, 10); + orderByColumns = Randomly.subset(table.getColumns()); + } + String originalDelete = gen.deleteStatement(table, predicate, orderByColumns, limit); + String transformedDelete = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); generatedQueryString = originalDelete; // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: From 0dea938244b227cc03c900982c7730a78544f002 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 15:33:50 +0800 Subject: [PATCH 121/132] Add existing insert/update errors to EET DML oracle expecteed errors --- src/sqlancer/common/gen/EETDMLGenerator.java | 9 ++++----- src/sqlancer/common/oracle/EETDMLOracle.java | 14 ++++++++++---- src/sqlancer/mysql/MySQLErrors.java | 2 +- src/sqlancer/mysql/MySQLOracleFactory.java | 6 +++++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 375877a63..82888e070 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -130,11 +130,10 @@ default String selectRowIdsStatement(T table) { * *

* When {@code limit} is non-null, the statement is ordered by {@code orderByColumns} followed by - * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even when - * the ordering columns tie), so the "first {@code limit}" rows are identical for the original and - * transformed statements. Varying the ordering columns exercises more access - * paths than the row id alone would. The caller must pass the same {@code orderByColumns} and {@code limit} - * to both statements; neither is transformed. + * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even + * when the ordering columns tie), so the "first {@code limit}" rows are identical for the original and transformed + * statements. Varying the ordering columns exercises more access paths than the row id alone would. The caller must + * pass the same {@code orderByColumns} and {@code limit} to both statements; neither is transformed. * * @param table * the table to delete from diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 792bc52c0..ce41039d6 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -99,11 +99,17 @@ public void check() throws SQLException { // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: // the ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak - // the column into the next iteration and cause cascading duplicate-column failures. - new SQLQueryAdapter(gen.addRowIdColumnStatement(table), true).execute(state); + // the column into the next iteration and cause cascading duplicate-column failures. The row-identity setup + // touches rows, so — like the DELETE itself — it can raise tolerated errors (e.g. functional-index maintenance + // truncation); such an error aborts the iteration (IgnoreMeException) rather than being reported as a bug. + if (!new SQLQueryAdapter(gen.addRowIdColumnStatement(table), errors, true).execute(state)) { + throw new IgnoreMeException(); + } try { // Stamp identifiers once, in autocommit mode, before both DELETEs run: both then observe the same rows. - new SQLQueryAdapter(gen.stampRowIdsStatement(table)).execute(state); + if (!new SQLQueryAdapter(gen.stampRowIdsStatement(table), errors).execute(state)) { + throw new IgnoreMeException(); + } Set originalSurvivors = executeDeleteAndSnapshot(table, originalDelete); Set transformedSurvivors = executeDeleteAndSnapshot(table, transformedDelete); @@ -113,7 +119,7 @@ public void check() throws SQLException { mismatchMessage(originalDelete, transformedDelete, originalSurvivors, transformedSurvivors)); } } finally { - new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), true).execute(state); + new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); } } diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 44316dc3f..0b6eb8284 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -66,7 +66,7 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { } public static List getDMLErrors() { - ArrayList errors = new ArrayList<>(); + ArrayList errors = new ArrayList<>(getInsertUpdateErrors()); // WHERE-clause type coercion (e.g. string -> number) is only a warning in SELECT but a hard error in // DELETE/UPDATE under strict sql_mode (MySQL 1292). A semantics-preserving transform may benignly change diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index d3d4b20b1..8b37efb13 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -99,7 +99,11 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) - .withRegex(MySQLErrors.getExpressionRegexErrors()).with(MySQLErrors.getDMLErrors()).build(); + .withRegex(MySQLErrors.getExpressionRegexErrors()) + // The DML statements and the row-identity setup (adding/stamping the auxiliary column) touch rows, + // so they can raise the full range of DML errors — e.g. functional-index maintenance truncation — + // beyond the SELECT-based expression errors. + .with(MySQLErrors.getDMLErrors()).build(); return new EETDMLOracle<>(globalState, gen, expectedErrors); } }; From 85d837a7215911cd11418e8e1caec6ca93c633f7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 8 Aug 2026 10:39:52 +0800 Subject: [PATCH 122/132] Increase generality of EETDMLGenerator.addRowIdColumnStatement --- src/sqlancer/common/gen/EETDMLGenerator.java | 15 ++++++++++++--- .../mysql/gen/MySQLExpressionGenerator.java | 6 ++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 82888e070..b56580a9d 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -77,7 +77,7 @@ public interface EETDMLGenerator, T extends AbstractTabl /** * SQL that assigns every existing row of {@code table} a distinct, stable identifier in the {@link #ROW_ID_COLUMN} - * column. DBMS-specific because it names the DBMS's UUID-generating function. + * column. For example, a 36-character UUID string. * * @param table * the table whose rows are stamped @@ -86,10 +86,19 @@ public interface EETDMLGenerator, T extends AbstractTabl */ String stampRowIdsStatement(T table); + /** + * The SQL type of the auxiliary {@link #ROW_ID_COLUMN} column. It must be able to hold the identifiers that + * {@link #stampRowIdsStatement} produces, so it belongs with that statement as the other half of the row-id + * representation. For example, {@code VARCHAR(36)} would fit a 36-character UUID string. + * + * @return the column type + */ + String rowIdColumnType(); + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- /** - * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}. + * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}, typed as {@link #rowIdColumnType}. * * @param table * the table to add the column to @@ -97,7 +106,7 @@ public interface EETDMLGenerator, T extends AbstractTabl * @return the SQL statement */ default String addRowIdColumnStatement(T table) { - return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " VARCHAR(36)"; + return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " " + rowIdColumnType(); } /** diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 5a694bd78..9a62c15ba 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -388,4 +388,10 @@ public String stampRowIdsStatement(MySQLTable table) { // column, delete, snapshot, transaction control) use EETDMLGenerator's defaults. return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); } + + @Override + public String rowIdColumnType() { + // Holds a 36-character UUID string produced by stampRowIdsStatement. + return "VARCHAR(36)"; + } } From 7b9b0549aa945ef583f94fed0b90bef7dad318b7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 19:48:07 +0800 Subject: [PATCH 123/132] Implement UPDATE support for EET DML oracle --- src/sqlancer/common/gen/EETDMLGenerator.java | 111 ++++++++--- src/sqlancer/common/oracle/EETDMLOracle.java | 174 +++++++++++++----- .../mysql/gen/MySQLExpressionGenerator.java | 18 +- 3 files changed, 231 insertions(+), 72 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index b56580a9d..9fb21ea35 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import sqlancer.common.ast.newast.Expression; import sqlancer.common.oracle.EETTransformer; @@ -19,7 +20,8 @@ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are - * produced. + * produced. The resulting state is compared as a full post-image (each surviving row's identifier and content column + * values), which covers every DML statement: a DELETE removes rows from it, an UPDATE changes values in it. * *

* Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. @@ -55,6 +57,15 @@ public interface EETDMLGenerator, T extends AbstractTabl */ E generateBooleanExpression(); + /** + * Generates a fresh set of {@code column = value} assignments over the current tables' columns, used as an UPDATE + * statement's SET clause. The columns are a random non-empty subset and each value is a fresh random expression; + * both the columns and their assigned expressions are transformed by the oracle. + * + * @return the assignments, as {@code (column, value expression)} pairs (at least one) + */ + List> generateSetAssignments(); + /** * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's * expressions into semantically equivalent ones. @@ -122,27 +133,32 @@ default String dropRowIdColumnStatement(T table) { } /** - * SQL that selects the {@link #ROW_ID_COLUMN} of every row of {@code table} (the surviving-row snapshot). + * SQL that reads back the full post-image of {@code table}: the {@link #ROW_ID_COLUMN} identifier and every content + * column of every surviving row, ordered by the (unique) identifier so the two statements' snapshots align + * row-for-row. + * + *

+ * This single value-level snapshot is the comparison surface for all DML statements: a DELETE removes rows from it, + * an UPDATE changes column values in it. Row identity alone (which the identifier already captures) would suffice + * for DELETE, but not for UPDATE, where the two runs could touch the same rows yet write different values. * * @param table * the table to snapshot * - * @return the SQL statement; its first result column must be the identifiers + * @return the SQL statement; its first result column is the identifier, followed by {@code table}'s content columns */ - default String selectRowIdsStatement(T table) { - return "SELECT " + ROW_ID_COLUMN + " FROM " + table.getName(); + default String selectPostImageStatement(T table) { + List selected = new ArrayList<>(); + selected.add(ROW_ID_COLUMN); + for (C column : table.getColumns()) { + selected.add(column.getName()); + } + return "SELECT " + String.join(", ", selected) + " FROM " + table.getName() + " ORDER BY " + ROW_ID_COLUMN; } /** * SQL that deletes the rows of {@code table} matching {@code predicate}, optionally limited to the first - * {@code limit} rows. - * - *

- * When {@code limit} is non-null, the statement is ordered by {@code orderByColumns} followed by - * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even - * when the ordering columns tie), so the "first {@code limit}" rows are identical for the original and transformed - * statements. Varying the ordering columns exercises more access paths than the row id alone would. The caller must - * pass the same {@code orderByColumns} and {@code limit} to both statements; neither is transformed. + * {@code limit} rows (see {@link #orderByLimitClause}). * * @param table * the table to delete from @@ -157,16 +173,67 @@ default String selectRowIdsStatement(T table) { * @return the SQL statement */ default String deleteStatement(T table, E predicate, List orderByColumns, Integer limit) { - String statement = "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); - if (limit != null) { - List orderBy = new ArrayList<>(); - for (C column : orderByColumns) { - orderBy.add(column.getName()); - } - orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above - statement += " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; + return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate) + + orderByLimitClause(orderByColumns, limit); + } + + /** + * SQL that updates the rows of {@code table} matching {@code predicate}, setting each column in {@code assignments} + * to its assigned value expression, optionally limited to the first {@code limit} rows (see + * {@link #orderByLimitClause}). + * + * @param table + * the table to update + * @param assignments + * the {@code (column, value expression)} pairs to assign; each value is rendered via {@link #asString} + * @param predicate + * the WHERE predicate; rendered via {@link #asString} + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty); only used when {@code limit} is + * non-null + * @param limit + * the maximum number of rows to update, or {@code null} for no limit + * + * @return the SQL statement + */ + default String updateStatement(T table, List> assignments, E predicate, List orderByColumns, + Integer limit) { + List setClauses = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + setClauses.add(assignment.getKey().getName() + " = " + asString(assignment.getValue())); + } + return "UPDATE " + table.getName() + " SET " + String.join(", ", setClauses) + " WHERE " + asString(predicate) + + orderByLimitClause(orderByColumns, limit); + } + + /** + * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and + * {@link #updateStatement}, or the empty string when {@code limit} is null. + * + *

+ * The rows are ordered by {@code orderByColumns} followed by {@link #ROW_ID_COLUMN} as a tiebreaker. Because the + * identifiers are unique, this is always a total order (even when the ordering columns tie), so the "first + * {@code limit}" rows are identical for the original and transformed statements. Varying the ordering columns + * exercises more access paths than the row id alone would. The caller must pass the same {@code orderByColumns} and + * {@code limit} to both statements; neither is transformed. + * + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty) + * @param limit + * the maximum number of rows, or {@code null} for no limit (yielding an empty clause) + * + * @return the {@code ORDER BY ... LIMIT n} clause, or the empty string when {@code limit} is null + */ + default String orderByLimitClause(List orderByColumns, Integer limit) { + if (limit == null) { + return ""; + } + List orderBy = new ArrayList<>(); + for (C column : orderByColumns) { + orderBy.add(column.getName()); } - return statement; + orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above + return " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; } /** diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index ce41039d6..3443c9756 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -1,11 +1,11 @@ package sqlancer.common.oracle; import java.sql.SQLException; -import java.util.HashSet; +import java.util.AbstractMap; +import java.util.ArrayList; import java.util.List; -import java.util.Set; +import java.util.Map; -import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLGlobalState; @@ -13,6 +13,7 @@ import sqlancer.common.gen.EETDMLGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; @@ -29,14 +30,17 @@ *

* Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row, and each statement is executed inside a transaction that is rolled back, so the two - * statements can be compared against the same starting state without permanently modifying the database. For a DELETE, - * the state is captured as the set of surviving row identifiers. Because rolling back a statement requires a - * transactional storage engine, the DBMS-specific setup must ensure only such engines are used while this oracle is - * active. + * statements can be compared against the same starting state without permanently modifying the database. The state is + * captured as a full post-image: each surviving row's identifier together with its content column values, ordered by + * the identifier. This single value-level surface covers every DML statement — a DELETE removes rows from it, an UPDATE + * changes values in it (row identity alone would suffice for DELETE, but not for UPDATE, which also transforms the + * written values). Because rolling back a statement requires a transactional storage engine, the DBMS-specific setup + * must ensure only such engines are used while this oracle is active. * *

- * Only DELETE is currently supported. Statement reduction is not yet implemented (there is no - * {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database reduction. + * DELETE and UPDATE are currently supported (one is chosen at random per check). Statement reduction is not yet + * implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database + * reduction. * * @param * the DBMS-specific expression class @@ -75,8 +79,9 @@ public void check() throws SQLException { if (tables.isEmpty()) { throw new IgnoreMeException(); } - // DELETE targets a single table, so operate on exactly one; confining the generator to it keeps the predicate - // from referencing another table's columns (which would render invalid single-table DML). + // A DML statement targets a single table, so operate on exactly one; confining the generator to it keeps the + // predicate and value expressions from referencing another table's columns (which would render invalid + // single-table DML). T table = Randomly.fromList(tables); gen = gen.setTablesAndColumns(new AbstractTables<>(List.of(table))); @@ -84,7 +89,7 @@ public void check() throws SQLException { // The WHERE predicate is evaluated in a boolean context. E transformedPredicate = transformer.transform(predicate, true); - // Optionally cap the DELETE with a LIMIT. The limit and its ordering (a random column subset, made a total + // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. Integer limit = null; @@ -93,30 +98,47 @@ public void check() throws SQLException { limit = (int) Randomly.getNotCachedInteger(0, 10); orderByColumns = Randomly.subset(table.getColumns()); } - String originalDelete = gen.deleteStatement(table, predicate, orderByColumns, limit); - String transformedDelete = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); - generatedQueryString = originalDelete; - - // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: - // the ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak - // the column into the next iteration and cause cascading duplicate-column failures. The row-identity setup - // touches rows, so — like the DELETE itself — it can raise tolerated errors (e.g. functional-index maintenance - // truncation); such an error aborts the iteration (IgnoreMeException) rather than being reported as a bug. + + String originalStatement; + String transformedStatement; + if (Randomly.getBoolean()) { + // UPDATE also transforms the written values: each SET value expression is transformed in a scalar context. + List> assignments = gen.generateSetAssignments(); + List> transformedAssignments = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + E transformedValue = transformer.transform(assignment.getValue(), false); + transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); + } + originalStatement = gen.updateStatement(table, assignments, predicate, orderByColumns, limit); + transformedStatement = gen.updateStatement(table, transformedAssignments, transformedPredicate, + orderByColumns, limit); + } else { + originalStatement = gen.deleteStatement(table, predicate, orderByColumns, limit); + transformedStatement = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); + } + generatedQueryString = originalStatement; + + // The post-image select reads the identifier plus every content column of the table, in that order. + int columnCount = table.getColumns().size() + 1; + + // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the + // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the + // column and cause cascading duplicate-column failures if (!new SQLQueryAdapter(gen.addRowIdColumnStatement(table), errors, true).execute(state)) { throw new IgnoreMeException(); } try { - // Stamp identifiers once, in autocommit mode, before both DELETEs run: both then observe the same rows. + // Stamp identifiers once, in autocommit mode, before both runs: both then observe the same rows. if (!new SQLQueryAdapter(gen.stampRowIdsStatement(table), errors).execute(state)) { throw new IgnoreMeException(); } - Set originalSurvivors = executeDeleteAndSnapshot(table, originalDelete); - Set transformedSurvivors = executeDeleteAndSnapshot(table, transformedDelete); + List> originalImage = executeAndSnapshotPostImage(table, originalStatement, columnCount); + List> transformedImage = executeAndSnapshotPostImage(table, transformedStatement, columnCount); - if (!originalSurvivors.equals(transformedSurvivors)) { + if (!originalImage.equals(transformedImage)) { throw new AssertionError( - mismatchMessage(originalDelete, transformedDelete, originalSurvivors, transformedSurvivors)); + mismatchMessage(originalStatement, transformedStatement, originalImage, transformedImage)); } } finally { new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); @@ -124,46 +146,102 @@ public void check() throws SQLException { } /** - * Executes {@code deleteStatement} inside a transaction that is always rolled back, and returns the set of row - * identifiers surviving the DELETE (the resulting database state). A DBMS error expected by the oracle aborts the - * whole check ({@link IgnoreMeException}) rather than being reported, matching {@link EETOracle}'s handling; an - * unexpected error surfaces as a bug ({@link AssertionError}, thrown by the query adapter). + * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: + * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A + * DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug or unexpected error + * surfaces as {@link AssertionError}. * * @param table - * the table being deleted from - * @param deleteStatement - * the DELETE statement to execute + * the table being modified + * @param statement + * the DML statement to execute + * @param columnCount + * the number of columns the post-image select returns (identifier plus content columns) * - * @return the set of row identifiers surviving the DELETE + * @return the post-image, as one string list (identifier followed by content column values) per surviving row * * @throws SQLException - * if a DBMS interaction fails + * if a DBMS interaction other than running {@code statement} fails; an error from {@code statement} + * itself instead surfaces as {@link IgnoreMeException} or {@link AssertionError} */ - private Set executeDeleteAndSnapshot(T table, String deleteStatement) throws SQLException { + private List> executeAndSnapshotPostImage(T table, String statement, int columnCount) + throws SQLException { new SQLQueryAdapter(gen.beginTransactionStatement()).execute(state); try { // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. - boolean succeeded = new SQLQueryAdapter(deleteStatement, errors).execute(state); + boolean succeeded = new SQLQueryAdapter(statement, errors).execute(state); if (!succeeded) { - // The DELETE hit an error the oracle tolerates; do not compare states (as EETOracle does for SELECT). + // The statement hit an error the oracle tolerates; do not compare states (as EETOracle does for + // SELECT). throw new IgnoreMeException(); } - return new HashSet<>( - ComparatorHelper.getResultSetFirstColumnAsString(gen.selectRowIdsStatement(table), errors, state)); + return snapshotPostImage(gen.selectPostImageStatement(table), columnCount); } finally { new SQLQueryAdapter(gen.rollbackTransactionStatement()).execute(state); } } - private static String mismatchMessage(String originalDelete, String transformedDelete, - Set originalSurvivors, Set transformedSurvivors) { + /** + * Reads the post-image produced by {@code selectStatement} into one string list per row (each column via + * {@code getString}). A DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug + * or unexpected error surfaces as {@link AssertionError}. + * + * @param selectStatement + * the post-image select to read; its columns are the identifier followed by the content columns + * @param columnCount + * the number of columns to read from each row + * + * @return the read rows, in the select's order + * + * @throws SQLException + * if cleanup fails (errors thrown elsewhere will always be rethrown as {@link IgnoreMeException} or + * {@link AssertionError}) + */ + private List> snapshotPostImage(String selectStatement, int columnCount) throws SQLException { + List> rows = new ArrayList<>(); + SQLQueryAdapter q = new SQLQueryAdapter(selectStatement, errors, true, + state.getOptions().canonicalizeSqlString()); + SQLancerResultSet result = null; + try { + result = q.executeAndGet(state); + if (result == null) { + throw new IgnoreMeException(); + } + while (result.next()) { + List row = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + row.add(result.getString(i)); + } + rows.add(row); + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw e; + } + Throwable current = e; + while (current != null) { + if (current.getMessage() != null && errors.errorIsExpected(current.getMessage())) { + throw new IgnoreMeException(); + } + current = current.getCause(); + } + throw new AssertionError(selectStatement, e); + } finally { + if (result != null && !result.isClosed()) { + result.close(); + } + } + return rows; + } + + private static String mismatchMessage(String originalStatement, String transformedStatement, + List> originalImage, List> transformedImage) { return new StringBuilder() - .append("-- The original and transformed DELETE statements left the database in different states") - .append(" (different sets of surviving rows):").append(System.lineSeparator()).append("-- original (") - .append(originalSurvivors.size()).append(" rows survive): ").append(originalDelete).append(';') - .append(System.lineSeparator()).append("-- transformed (").append(transformedSurvivors.size()) - .append(" rows survive): ").append(transformedDelete).append(';').append(System.lineSeparator()) - .toString(); + .append("-- The original and transformed statements left the database in different states") + .append(" (different post-images):").append(System.lineSeparator()).append("-- original (") + .append(originalImage.size()).append(" rows): ").append(originalStatement).append(';') + .append(System.lineSeparator()).append("-- transformed (").append(transformedImage.size()) + .append(" rows): ").append(transformedStatement).append(';').append(System.lineSeparator()).toString(); } @Override diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9a62c15ba..9fbaef12a 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -1,7 +1,9 @@ package sqlancer.mysql.gen; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -254,6 +256,18 @@ public MySQLExpression generateBooleanExpression() { return generateExpression(); } + @Override + public List> generateSetAssignments() { + List> assignments = new ArrayList<>(); + for (MySQLColumn column : Randomly.nonEmptySubset(columns)) { + // As with the normal UPDATE workload, the value is an arbitrary expression (not type-matched to the + // column); + // any resulting type/range error is on the oracle's expected-error allow-list. + assignments.add(new AbstractMap.SimpleEntry<>(column, generateExpression())); + } + return assignments; + } + @Override public MySQLSelect generateSelect() { return new MySQLSelect(); @@ -384,8 +398,8 @@ public String asString(MySQLExpression expr) { @Override public String stampRowIdsStatement(MySQLTable table) { // MySQL's UUID() gives each existing row a distinct value in a single statement. Stamping happens once, before - // both rolled-back DELETE runs, so both observe identical identifiers; the standard-SQL statements (add/drop - // column, delete, snapshot, transaction control) use EETDMLGenerator's defaults. + // both rolled-back statement runs, so both observe identical identifiers; the standard-SQL statements (add/drop + // column, delete/update, snapshot, transaction control) use EETDMLGenerator's defaults. return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); } From 1e53207c5770081e746e67a9ee1e5dac6abc5d94 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 20:11:53 +0800 Subject: [PATCH 124/132] Add row discrepancy information to test case logs for EET DML oracle --- src/sqlancer/common/oracle/EETDMLOracle.java | 65 +++++++++++++++++--- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 3443c9756..fbb2d2d3b 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -3,8 +3,12 @@ import java.sql.SQLException; import java.util.AbstractMap; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -61,6 +65,7 @@ public class EETDMLOracle, S extends AbstractSchema transformer; private final ExpectedErrors errors; + private static final int MAX_DIFF_ROWS_REPORTED = 10; // max differing post-image rows displayed in report log private String generatedQueryString; public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { @@ -137,8 +142,8 @@ public void check() throws SQLException { List> transformedImage = executeAndSnapshotPostImage(table, transformedStatement, columnCount); if (!originalImage.equals(transformedImage)) { - throw new AssertionError( - mismatchMessage(originalStatement, transformedStatement, originalImage, transformedImage)); + throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, originalImage, + transformedImage)); } } finally { new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); @@ -234,14 +239,56 @@ private List> snapshotPostImage(String selectStatement, int columnC return rows; } - private static String mismatchMessage(String originalStatement, String transformedStatement, + private String mismatchMessage(T table, String originalStatement, String transformedStatement, List> originalImage, List> transformedImage) { - return new StringBuilder() - .append("-- The original and transformed statements left the database in different states") - .append(" (different post-images):").append(System.lineSeparator()).append("-- original (") - .append(originalImage.size()).append(" rows): ").append(originalStatement).append(';') - .append(System.lineSeparator()).append("-- transformed (").append(transformedImage.size()) - .append(" rows): ").append(transformedStatement).append(';').append(System.lineSeparator()).toString(); + List header = new ArrayList<>(); + header.add(EETDMLGenerator.ROW_ID_COLUMN); + for (C column : table.getColumns()) { + header.add(column.getName()); + } + + Map> originalByRowId = indexByRowId(originalImage); + Map> transformedByRowId = indexByRowId(transformedImage); + Set allRowIds = new TreeSet<>(); + allRowIds.addAll(originalByRowId.keySet()); + allRowIds.addAll(transformedByRowId.keySet()); + + String nl = System.lineSeparator(); + StringBuilder message = new StringBuilder() + .append("-- The original and transformed statements left the database in different states.").append(nl) + .append("-- original: ").append(originalStatement).append(';').append(nl).append("-- transformed: ") + .append(transformedStatement).append(';').append(nl).append("-- differing post-image rows (") + .append(String.join(", ", header)).append("):").append(nl); + int shown = 0; + for (String rowId : allRowIds) { + List originalRow = originalByRowId.get(rowId); + List transformedRow = transformedByRowId.get(rowId); + if (Objects.equals(originalRow, transformedRow)) { + continue; + } + if (shown == MAX_DIFF_ROWS_REPORTED) { + message.append("-- ... (further differences omitted)").append(nl); + break; + } + message.append("-- original: ").append(renderRow(originalRow)).append(nl); + message.append("-- transformed: ").append(renderRow(transformedRow)).append(nl); + shown++; + } + return message.toString(); + } + + // Indexes a post-image by its row identifier (the first column of each row) + private static Map> indexByRowId(List> image) { + Map> byRowId = new LinkedHashMap<>(); + for (List row : image) { + byRowId.put(row.get(0), row); + } + return byRowId; + } + + // Renders a post-image row for the finding message, or "(row absent)" when the row is missing on that side + private static String renderRow(List row) { + return row == null ? "(row absent)" : row.toString(); } @Override From 63f50b826a7b217ed12f4fdd8aeb10613ea166df Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 12 Aug 2026 09:28:31 +0800 Subject: [PATCH 125/132] Remove hard-coding of EET DML rowId column being at index 0; derive it instead --- src/sqlancer/common/gen/EETDMLGenerator.java | 26 ++++++++++++++++---- src/sqlancer/common/oracle/EETDMLOracle.java | 21 +++++++--------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 9fb21ea35..0c84901f3 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -145,15 +145,31 @@ default String dropRowIdColumnStatement(T table) { * @param table * the table to snapshot * - * @return the SQL statement; its first result column is the identifier, followed by {@code table}'s content columns + * @return the SQL statement; its result columns are those of {@link #postImageColumns}, in that order */ default String selectPostImageStatement(T table) { - List selected = new ArrayList<>(); - selected.add(ROW_ID_COLUMN); + return "SELECT " + String.join(", ", postImageColumns(table)) + " FROM " + table.getName() + " ORDER BY " + + ROW_ID_COLUMN; + } + + /** + * The columns a post-image row consists of, in the order {@link #selectPostImageStatement} returns them: the + * {@link #ROW_ID_COLUMN} identifier followed by {@code table}'s content columns. This is the sole definition of the + * post-image layout, so a consumer can find the identifier's position by looking up {@link #ROW_ID_COLUMN} here + * rather than assuming one. + * + * @param table + * the table being snapshot + * + * @return the post-image column names, in order + */ + default List postImageColumns(T table) { + List columns = new ArrayList<>(); + columns.add(ROW_ID_COLUMN); for (C column : table.getColumns()) { - selected.add(column.getName()); + columns.add(column.getName()); } - return "SELECT " + String.join(", ", selected) + " FROM " + table.getName() + " ORDER BY " + ROW_ID_COLUMN; + return columns; } /** diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index fbb2d2d3b..eb453d9b6 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -123,8 +123,7 @@ public void check() throws SQLException { } generatedQueryString = originalStatement; - // The post-image select reads the identifier plus every content column of the table, in that order. - int columnCount = table.getColumns().size() + 1; + int columnCount = gen.postImageColumns(table).size(); // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the @@ -241,14 +240,12 @@ private List> snapshotPostImage(String selectStatement, int columnC private String mismatchMessage(T table, String originalStatement, String transformedStatement, List> originalImage, List> transformedImage) { - List header = new ArrayList<>(); - header.add(EETDMLGenerator.ROW_ID_COLUMN); - for (C column : table.getColumns()) { - header.add(column.getName()); - } + List header = gen.postImageColumns(table); + // Where the identifier sits within a post-image row, per the layout the generator defines + int rowIdIndex = header.indexOf(EETDMLGenerator.ROW_ID_COLUMN); - Map> originalByRowId = indexByRowId(originalImage); - Map> transformedByRowId = indexByRowId(transformedImage); + Map> originalByRowId = indexByRowId(originalImage, rowIdIndex); + Map> transformedByRowId = indexByRowId(transformedImage, rowIdIndex); Set allRowIds = new TreeSet<>(); allRowIds.addAll(originalByRowId.keySet()); allRowIds.addAll(transformedByRowId.keySet()); @@ -277,11 +274,11 @@ private String mismatchMessage(T table, String originalStatement, String transfo return message.toString(); } - // Indexes a post-image by its row identifier (the first column of each row) - private static Map> indexByRowId(List> image) { + // Indexes a post-image by its row identifier, which each row holds at rowIdIndex + private static Map> indexByRowId(List> image, int rowIdIndex) { Map> byRowId = new LinkedHashMap<>(); for (List row : image) { - byRowId.put(row.get(0), row); + byRowId.put(row.get(rowIdIndex), row); } return byRowId; } From 1ce0341485a5052d9055a120cc3bf98321c3d0ee Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 12 Aug 2026 09:53:21 +0800 Subject: [PATCH 126/132] Refactor EET DML statement generation to use dedicated methods for update and delete --- src/sqlancer/common/oracle/EETDMLOracle.java | 86 ++++++++++++++++---- 1 file changed, 69 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index eb453d9b6..4731c04ed 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -104,23 +104,11 @@ public void check() throws SQLException { orderByColumns = Randomly.subset(table.getColumns()); } - String originalStatement; - String transformedStatement; - if (Randomly.getBoolean()) { - // UPDATE also transforms the written values: each SET value expression is transformed in a scalar context. - List> assignments = gen.generateSetAssignments(); - List> transformedAssignments = new ArrayList<>(); - for (Map.Entry assignment : assignments) { - E transformedValue = transformer.transform(assignment.getValue(), false); - transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); - } - originalStatement = gen.updateStatement(table, assignments, predicate, orderByColumns, limit); - transformedStatement = gen.updateStatement(table, transformedAssignments, transformedPredicate, - orderByColumns, limit); - } else { - originalStatement = gen.deleteStatement(table, predicate, orderByColumns, limit); - transformedStatement = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); - } + StatementPair statements = Randomly.getBoolean() + ? generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit) + : generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit); + String originalStatement = statements.original; + String transformedStatement = statements.transformed; generatedQueryString = originalStatement; int columnCount = gen.postImageColumns(table).size(); @@ -149,6 +137,70 @@ public void check() throws SQLException { } } + /** + * A DML statement and its transformed counterpart, which must leave the database in the same state. + */ + private static final class StatementPair { + private final String original; + private final String transformed; + + StatementPair(String original, String transformed) { + this.original = original; + this.transformed = transformed; + } + } + + /** + * Generates an UPDATE and its transformed counterpart. Besides the WHERE predicate, UPDATE also transforms the + * written values: each SET value expression is transformed in a scalar context. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the statement, empty if it is not capped by a limit + * @param limit + * the maximum number of rows to modify, or {@code null} for no limit + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateUpdateStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { + List> assignments = gen.generateSetAssignments(); + List> transformedAssignments = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + E transformedValue = transformer.transform(assignment.getValue(), false); + transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); + } + return new StatementPair(gen.updateStatement(table, assignments, predicate, orderByColumns, limit), + gen.updateStatement(table, transformedAssignments, transformedPredicate, orderByColumns, limit)); + } + + /** + * Generates a DELETE and its transformed counterpart, which differ only in their WHERE predicate. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the statement, empty if it is not capped by a limit + * @param limit + * the maximum number of rows to modify, or {@code null} for no limit + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateDeleteStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { + return new StatementPair(gen.deleteStatement(table, predicate, orderByColumns, limit), + gen.deleteStatement(table, transformedPredicate, orderByColumns, limit)); + } + /** * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A From 4cf31d15f2f841686365f0167ad489222eb4b90f Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 13 Aug 2026 13:40:00 +0800 Subject: [PATCH 127/132] Implement INSERT support for EET DML oracle --- src/sqlancer/common/gen/EETDMLGenerator.java | 72 +++++++++++++++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 61 ++++++++++++---- .../mysql/gen/MySQLExpressionGenerator.java | 16 +++++ 3 files changed, 130 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 0c84901f3..c10c59b5a 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -19,9 +19,9 @@ *

* Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run - * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are - * produced. The resulting state is compared as a full post-image (each surviving row's identifier and content column - * values), which covers every DML statement: a DELETE removes rows from it, an UPDATE changes values in it. + * (each in a rolled-back transaction), so both executions observe the same identifiers. The resulting state is compared + * as a full post-image (each surviving row's identifier and content column values), which covers any of the three DML + * statements (DELETE, UPDATE, INSERT). * *

* Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. @@ -66,6 +66,15 @@ public interface EETDMLGenerator, T extends AbstractTabl */ List> generateSetAssignments(); + /** + * Generates a fresh value expression for each content column of the current table, used as an INSERT statement's + * inserted values. The returned expressions are positionally aligned with {@link AbstractTable#getColumns()}, and + * each is transformed by the oracle. + * + * @return one fresh random value expression per content column, in {@link AbstractTable#getColumns()} order + */ + List generateInsertValues(); + /** * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's * expressions into semantically equivalent ones. @@ -106,6 +115,17 @@ public interface EETDMLGenerator, T extends AbstractTabl */ String rowIdColumnType(); + /** + * A SQL expression, evaluated once per source row of an {@code INSERT ... SELECT}, that derives the inserted row's + * {@link #ROW_ID_COLUMN} value from the source row's identifier. It must be deterministic (so both the original and + * transformed statements assign the same identifiers), unique per source row, and distinct from every existing + * identifier (so an inserted row never collides with the source row it was derived from in the post-image). DBMS- + * specific because it names a suitable derivation function (e.g. a hash of the source identifier). + * + * @return the SQL expression deriving an inserted row's identifier from the source row's {@link #ROW_ID_COLUMN} + */ + String insertedRowIdExpression(); + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- /** @@ -139,8 +159,9 @@ default String dropRowIdColumnStatement(T table) { * *

* This single value-level snapshot is the comparison surface for all DML statements: a DELETE removes rows from it, - * an UPDATE changes column values in it. Row identity alone (which the identifier already captures) would suffice - * for DELETE, but not for UPDATE, where the two runs could touch the same rows yet write different values. + * an UPDATE changes column values in it, an INSERT adds rows to it. Row identity alone (which the identifier + * already captures) would suffice for DELETE, but not for UPDATE, where the two runs could touch the same rows yet + * write different values. * * @param table * the table to snapshot @@ -222,6 +243,47 @@ default String updateStatement(T table, List> assignments, E pre + orderByLimitClause(orderByColumns, limit); } + /** + * SQL that inserts a new row into {@code table} for each source row (optionally filtered by {@code predicate}), + * setting each content column to its corresponding value in {@code values}. + * + *

+ * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value + * expressions reference the table's columns (the transformer injects column references into its equivalent + * sub-expressions), which are legal in a {@code SELECT} but not in a {@code VALUES} clause. Each inserted row's + * {@link #ROW_ID_COLUMN} is derived from its source row via {@link #insertedRowIdExpression()}, giving it a + * deterministic identifier that is unique and distinct from every existing one, so the two statements' post-images + * align (and inserted rows never collide with their source rows). + * + * @param table + * the table to insert into + * @param values + * one value expression per content column, positionally aligned with {@link AbstractTable#getColumns()}; + * each is rendered via {@link #asString} + * @param predicate + * the WHERE predicate filtering the source rows, or {@code null} to insert from every source row; + * rendered via {@link #asString} + * + * @return the SQL statement + */ + default String insertStatement(T table, List values, E predicate) { + List columnNames = new ArrayList<>(); + columnNames.add(ROW_ID_COLUMN); + List selectItems = new ArrayList<>(); + selectItems.add(insertedRowIdExpression()); + List columns = table.getColumns(); + for (int i = 0; i < columns.size(); i++) { + columnNames.add(columns.get(i).getName()); + selectItems.add(asString(values.get(i))); + } + String statement = "INSERT INTO " + table.getName() + " (" + String.join(", ", columnNames) + ") SELECT " + + String.join(", ", selectItems) + " FROM " + table.getName(); + if (predicate != null) { + statement += " WHERE " + asString(predicate); + } + return statement; + } + /** * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and * {@link #updateStatement}, or the empty string when {@code limit} is null. diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 4731c04ed..ba9b9d6c7 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -9,6 +9,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import java.util.function.Supplier; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -37,13 +38,15 @@ * statements can be compared against the same starting state without permanently modifying the database. The state is * captured as a full post-image: each surviving row's identifier together with its content column values, ordered by * the identifier. This single value-level surface covers every DML statement — a DELETE removes rows from it, an UPDATE - * changes values in it (row identity alone would suffice for DELETE, but not for UPDATE, which also transforms the - * written values). Because rolling back a statement requires a transactional storage engine, the DBMS-specific setup - * must ensure only such engines are used while this oracle is active. + * changes values in it, an INSERT adds rows to it (row identity alone would suffice for DELETE, but not for UPDATE, + * which also transforms the written values). Because rolling back a statement requires a transactional storage engine, + * the DBMS-specific setup must ensure only such engines are used while this oracle is active. * *

- * DELETE and UPDATE are currently supported (one is chosen at random per check). Statement reduction is not yet - * implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database + * DELETE, UPDATE and INSERT are currently supported (one is chosen at random per check). INSERT uses the + * {@code INSERT ... SELECT} form so its transformed value expressions may reference columns; each inserted row is given + * a deterministic identifier derived from its source row so the two runs' post-images align. Statement reduction is not + * yet implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database * reduction. * * @param @@ -97,16 +100,16 @@ public void check() throws SQLException { // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. - Integer limit = null; - List orderByColumns = List.of(); - if (Randomly.getBoolean()) { - limit = (int) Randomly.getNotCachedInteger(0, 10); - orderByColumns = Randomly.subset(table.getColumns()); - } + boolean withLimit = Randomly.getBoolean(); + Integer limit = withLimit ? (int) Randomly.getNotCachedInteger(0, 10) : null; + List orderByColumns = withLimit ? Randomly.subset(table.getColumns()) : List.of(); - StatementPair statements = Randomly.getBoolean() - ? generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit) - : generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit); + // Generators for the different kinds of statement this oracle supports. One is chosen at random per check + List> statementGenerators = List.of( + () -> generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit), + () -> generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit), + () -> generateInsertStatements(table, predicate, transformedPredicate)); + StatementPair statements = Randomly.fromList(statementGenerators).get(); String originalStatement = statements.original; String transformedStatement = statements.transformed; generatedQueryString = originalStatement; @@ -201,6 +204,36 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform gen.deleteStatement(table, transformedPredicate, orderByColumns, limit)); } + /** + * Generates an {@code INSERT ... SELECT} and its transformed counterpart. Besides the WHERE predicate, which + * filters the source rows and is optional here, INSERT also transforms each inserted value in a scalar context. + * + *

+ * Unlike DELETE and UPDATE, no limit is applied: {@link EETDMLGenerator#insertStatement} renders no ordering or + * limit, so one row is inserted per source row the predicate keeps. Nothing about INSERT rules a limit out — its + * source SELECT could carry the same ordering and limit the other statement kinds use, and the two runs would still + * read the same source rows — it is just not generated. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate) { + List values = gen.generateInsertValues(); + List transformedValues = new ArrayList<>(); + for (E value : values) { + transformedValues.add(transformer.transform(value, false)); + } + boolean withPredicate = Randomly.getBoolean(); + return new StatementPair(gen.insertStatement(table, values, withPredicate ? predicate : null), + gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null)); + } + /** * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9fbaef12a..baea11f65 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -268,6 +268,14 @@ public List> generateSetAssignments() { return assignments; } + @Override + public List generateInsertValues() { + // One value per content column, in schema order (aligned with the INSERT column list). As with the normal + // INSERT workload, each value is an arbitrary expression (not type-matched to the column); any resulting + // type/range/constraint error is on the oracle's expected-error allow-list. + return columns.stream().map(c -> generateExpression()).collect(Collectors.toList()); + } + @Override public MySQLSelect generateSelect() { return new MySQLSelect(); @@ -408,4 +416,12 @@ public String rowIdColumnType() { // Holds a 36-character UUID string produced by stampRowIdsStatement. return "VARCHAR(36)"; } + + @Override + public String insertedRowIdExpression() { + // The source row's identifier with its dashes removed: deterministic (identical across both runs) and unique + // per + // source row. Fits the identifier column's VARCHAR(36). + return String.format("REPLACE(%s, '-', '')", ROW_ID_COLUMN); + } } From 7ead2e762106cdab3c71be9b094b1c42b033afb7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 13 Aug 2026 13:50:17 +0800 Subject: [PATCH 128/132] Add support for LIMIT on source rows of INSERT ... SELECT statements in EET --- src/sqlancer/common/gen/EETDMLGenerator.java | 16 ++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 53 ++++++++++++++------ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index c10c59b5a..557220d06 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -245,7 +245,8 @@ default String updateStatement(T table, List> assignments, E pre /** * SQL that inserts a new row into {@code table} for each source row (optionally filtered by {@code predicate}), - * setting each content column to its corresponding value in {@code values}. + * setting each content column to its corresponding value in {@code values}, optionally limited to the first + * {@code limit} source rows (see {@link #orderByLimitClause}). * *

* The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value @@ -263,10 +264,15 @@ default String updateStatement(T table, List> assignments, E pre * @param predicate * the WHERE predicate filtering the source rows, or {@code null} to insert from every source row; * rendered via {@link #asString} + * @param orderByColumns + * the columns to order the source rows by before the row-id tiebreaker (may be empty); only used when + * {@code limit} is non-null + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit * * @return the SQL statement */ - default String insertStatement(T table, List values, E predicate) { + default String insertStatement(T table, List values, E predicate, List orderByColumns, Integer limit) { List columnNames = new ArrayList<>(); columnNames.add(ROW_ID_COLUMN); List selectItems = new ArrayList<>(); @@ -281,12 +287,12 @@ default String insertStatement(T table, List values, E predicate) { if (predicate != null) { statement += " WHERE " + asString(predicate); } - return statement; + return statement + orderByLimitClause(orderByColumns, limit); } /** - * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and - * {@link #updateStatement}, or the empty string when {@code limit} is null. + * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement}, + * {@link #updateStatement} and {@link #insertStatement}, or the empty string when {@code limit} is null. * *

* The rows are ordered by {@code orderByColumns} followed by {@link #ROW_ID_COLUMN} as a tiebreaker. Because the diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index ba9b9d6c7..02bff8165 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -9,7 +9,6 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; -import java.util.function.Supplier; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -100,16 +99,18 @@ public void check() throws SQLException { // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. - boolean withLimit = Randomly.getBoolean(); - Integer limit = withLimit ? (int) Randomly.getNotCachedInteger(0, 10) : null; - List orderByColumns = withLimit ? Randomly.subset(table.getColumns()) : List.of(); + Integer limit = null; + List orderByColumns = List.of(); + if (Randomly.getBoolean()) { + limit = (int) Randomly.getNotCachedInteger(0, 10); + orderByColumns = Randomly.subset(table.getColumns()); + } // Generators for the different kinds of statement this oracle supports. One is chosen at random per check - List> statementGenerators = List.of( - () -> generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit), - () -> generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit), - () -> generateInsertStatements(table, predicate, transformedPredicate)); - StatementPair statements = Randomly.fromList(statementGenerators).get(); + List> statementGenerators = List.of(this::generateDeleteStatements, + this::generateUpdateStatements, this::generateInsertStatements); + StatementPair statements = Randomly.fromList(statementGenerators).generate(table, predicate, + transformedPredicate, orderByColumns, limit); String originalStatement = statements.original; String transformedStatement = statements.transformed; generatedQueryString = originalStatement; @@ -140,6 +141,22 @@ public void check() throws SQLException { } } + /** + * Generates a DML statement of one kind together with its transformed counterpart. The kinds share this signature + * so the oracle can pick one of them at random per check. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ + @FunctionalInterface + private interface DMLStatementGenerator { + StatementPair generate(T table, E predicate, E transformedPredicate, List orderByColumns, Integer limit); + } + /** * A DML statement and its transformed counterpart, which must leave the database in the same state. */ @@ -209,10 +226,7 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform * filters the source rows and is optional here, INSERT also transforms each inserted value in a scalar context. * *

- * Unlike DELETE and UPDATE, no limit is applied: {@link EETDMLGenerator#insertStatement} renders no ordering or - * limit, so one row is inserted per source row the predicate keeps. Nothing about INSERT rules a limit out — its - * source SELECT could carry the same ordering and limit the other statement kinds use, and the two runs would still - * read the same source rows — it is just not generated. + * The ordering and limit cap the source rows the statement reads, so it inserts one row per source row kept. * * @param table * the table being modified @@ -220,18 +234,25 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform * the WHERE predicate of the original statement * @param transformedPredicate * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the source rows, empty if the statement is not capped by a limit + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit * * @return the original statement together with its transformed counterpart */ - private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate) { + private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { List values = gen.generateInsertValues(); List transformedValues = new ArrayList<>(); for (E value : values) { transformedValues.add(transformer.transform(value, false)); } boolean withPredicate = Randomly.getBoolean(); - return new StatementPair(gen.insertStatement(table, values, withPredicate ? predicate : null), - gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null)); + return new StatementPair( + gen.insertStatement(table, values, withPredicate ? predicate : null, orderByColumns, limit), + gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null, + orderByColumns, limit)); } /** From f46934ecb6ec699c0315631e38e71a8e97e8b27f Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 10:11:10 +0800 Subject: [PATCH 129/132] Improve robustness of MySQL EET DML by ensuring InnoDB engine is always chosen --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 0746056ad..d27d6e681 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -164,7 +164,15 @@ public static List getRandomTableOptions() { } private void appendTableOptions() { - List tableOptions = TableOptions.getRandomTableOptions(); + List tableOptions = new ArrayList<>(TableOptions.getRandomTableOptions()); + // The EET DML oracle rolls back each statement to compare database states, which requires a transactional + // engine. The ENGINE option already forces InnoDB when the oracle is active (see the ENGINE case below), but it + // is only emitted when randomly chosen; otherwise the table would inherit the server's default engine, which is + // not guaranteed transactional. Force the option to always be present so the engine is never left to the + // server default. + if (globalState.usesEETDML() && !tableOptions.contains(TableOptions.ENGINE)) { + tableOptions.add(TableOptions.ENGINE); + } int i = 0; for (TableOptions o : tableOptions) { if (i++ != 0) { From f1e65caeb45e122bd968fdf7f7a6e783edd122d2 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 14 Aug 2026 10:27:43 +0800 Subject: [PATCH 130/132] Clarify EET DML insertStatement documentation --- src/sqlancer/common/gen/EETDMLGenerator.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 557220d06..9dbc9f0d5 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -249,12 +249,12 @@ default String updateStatement(T table, List> assignments, E pre * {@code limit} source rows (see {@link #orderByLimitClause}). * *

- * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value - * expressions reference the table's columns (the transformer injects column references into its equivalent - * sub-expressions), which are legal in a {@code SELECT} but not in a {@code VALUES} clause. Each inserted row's - * {@link #ROW_ID_COLUMN} is derived from its source row via {@link #insertedRowIdExpression()}, giving it a - * deterministic identifier that is unique and distinct from every existing one, so the two statements' post-images - * align (and inserted rows never collide with their source rows). + * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because it reuses the source-row + * model already shared by {@link #deleteStatement} and {@link #updateStatement}, and because it offers two kinds of + * transformable expression in one statement (the inserted values and the WHERE predicate) rather than the values + * alone. Each inserted row's {@link #ROW_ID_COLUMN} is derived from its source row via + * {@link #insertedRowIdExpression()}, giving it a deterministic identifier that is unique and distinct from every + * existing one, so the two statements' post-images align (and inserted rows never collide with their source rows). * * @param table * the table to insert into From 3c707e66c0fef2ba22cd29aa724bab54b4872549 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 31 Jul 2026 11:44:33 +0800 Subject: [PATCH 131/132] Add reproducer to EET DML --- src/sqlancer/common/oracle/EETDMLOracle.java | 185 ++++++++++++++++--- 1 file changed, 158 insertions(+), 27 deletions(-) diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 02bff8165..77a8b528a 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -12,6 +12,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.SQLGlobalState; import sqlancer.common.ast.newast.Expression; import sqlancer.common.gen.EETDMLGenerator; @@ -44,9 +45,9 @@ *

* DELETE, UPDATE and INSERT are currently supported (one is chosen at random per check). INSERT uses the * {@code INSERT ... SELECT} form so its transformed value expressions may reference columns; each inserted row is given - * a deterministic identifier derived from its source row so the two runs' post-images align. Statement reduction is not - * yet implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database - * reduction. + * a deterministic identifier derived from its source row so the two runs' post-images align. To support reduction, a + * {@link Reproducer} replays the whole comparison (adding and stamping the row-identifier column, running both + * statements in rolled-back transactions and comparing the post-images) against the reduced database. * * @param * the DBMS-specific expression class @@ -69,6 +70,93 @@ public class EETDMLOracle, S extends AbstractSchema reproducer; + + // The SQL and metadata to run and observe one DML comparison, captured as strings so a reproducer can replay it + // against a reduced database without the generator or live schema objects. + private static final class ComparisonQueries { + private final String originalStatement; + private final String transformedStatement; + private final String addRowIdColumn; + private final String stampRowIds; + private final String beginTransaction; + private final String rollback; + private final String dropRowIdColumn; + private final String selectPostImage; + private final int columnCount; + + ComparisonQueries(String originalStatement, String transformedStatement, String addRowIdColumn, + String stampRowIds, String beginTransaction, String rollback, String dropRowIdColumn, + String selectPostImage, int columnCount) { + this.originalStatement = originalStatement; + this.transformedStatement = transformedStatement; + this.addRowIdColumn = addRowIdColumn; + this.stampRowIds = stampRowIds; + this.beginTransaction = beginTransaction; + this.rollback = rollback; + this.dropRowIdColumn = dropRowIdColumn; + this.selectPostImage = selectPostImage; + this.columnCount = columnCount; + } + } + + // The post-images the original and transformed statements produced, compared for equality to detect the bug. + private static final class PostImages { + private final List> original; + private final List> transformed; + + PostImages(List> original, List> transformed) { + this.original = original; + this.transformed = transformed; + } + } + + // Reproduces a post-image mismatch against the reduced database. Unlike EETOracle's comparison reproducer this does + // not extend AbstractComparisonReproducer: the two sides are not independent, because the row-id stamping (UUID()) + // must run once so both observe the same rows, so both post-images are computed together. + private final class EETDMLReproducer implements Reproducer { + private final ComparisonQueries queries; + + EETDMLReproducer(ComparisonQueries queries) { + this.queries = queries; + } + + @Override + public boolean bugStillTriggers(G globalState) { + PostImages images; + try { + images = computePostImages(globalState, queries); + } catch (AssertionError | SQLException | RuntimeException e) { + // any failure re-running the comparison means this reduced database no longer shows the mismatch + return false; + } + return !images.original.equals(images.transformed); + } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + sb.append("-- On the database set up by the statements above, the following statements leave the database" + + " in different states:").append(System.lineSeparator()); + renderStatementLines(sb, queries); + return sb.toString(); + } + } + + // Builds the reproducer for an unexpected DBMS error, which replays the whole comparison and checks the same error + // still fires. + private UnexpectedErrorReproducer errorReproducer(ComparisonQueries queries, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> computePostImages(globalState, queries); + StringBuilder sb = new StringBuilder(); + renderStatementLines(sb, queries); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + + // Renders the failing statements as commented lines, shared by the mismatch and the unexpected-error reproducers. + private static void renderStatementLines(StringBuilder sb, ComparisonQueries queries) { + sb.append("-- original: ").append(queries.originalStatement).append(';').append(System.lineSeparator()); + sb.append("-- transformed: ").append(queries.transformedStatement).append(';').append(System.lineSeparator()); + } public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { @@ -82,6 +170,7 @@ public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expect @Override public void check() throws SQLException { + reproducer = null; List tables = state.getSchema().getDatabaseTables(); if (tables.isEmpty()) { throw new IgnoreMeException(); @@ -115,29 +204,63 @@ public void check() throws SQLException { String transformedStatement = statements.transformed; generatedQueryString = originalStatement; - int columnCount = gen.postImageColumns(table).size(); + // Capture, as strings, everything needed to run and observe this comparison: the two statements plus the + // auxiliary-column setup, per-run snapshot and teardown. A reproducer replays these against a reduced database, + // where the live generator and schema objects no longer apply. + ComparisonQueries queries = new ComparisonQueries(originalStatement, transformedStatement, + gen.addRowIdColumnStatement(table), gen.stampRowIdsStatement(table), gen.beginTransactionStatement(), + gen.rollbackTransactionStatement(), gen.dropRowIdColumnStatement(table), + gen.selectPostImageStatement(table), gen.postImageColumns(table).size()); + PostImages images; + try { + images = computePostImages(state, queries); + } catch (AssertionError unexpectedError) { + reproducer = errorReproducer(queries, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + reproducer = new EETDMLReproducer(queries); + if (!images.original.equals(images.transformed)) { + throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, images.original, + images.transformed)); + } + } + + /** + * Runs the whole comparison against {@code globalState}: adds and stamps the row-identifier column once (so both + * runs observe the same rows), snapshots the post-image each statement produces (each inside a rolled-back + * transaction), and drops the column. Both {@link #check()} and the reproducers call this, the former against the + * live database and the latter against a reduced one. A DBMS error the oracle tolerates aborts with + * {@link IgnoreMeException}; an oracle logic bug or unexpected error surfaces as {@link AssertionError}. + * + * @param globalState + * the state whose connection the comparison runs against + * @param queries + * the statements and auxiliary SQL to run + * + * @return the post-images the original and transformed statements produced + * + * @throws SQLException + * if a DBMS interaction fails + */ + private PostImages computePostImages(G globalState, ComparisonQueries queries) throws SQLException { // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the // column and cause cascading duplicate-column failures - if (!new SQLQueryAdapter(gen.addRowIdColumnStatement(table), errors, true).execute(state)) { + if (!new SQLQueryAdapter(queries.addRowIdColumn, errors, true).execute(globalState)) { throw new IgnoreMeException(); } try { // Stamp identifiers once, in autocommit mode, before both runs: both then observe the same rows. - if (!new SQLQueryAdapter(gen.stampRowIdsStatement(table), errors).execute(state)) { + if (!new SQLQueryAdapter(queries.stampRowIds, errors).execute(globalState)) { throw new IgnoreMeException(); } - - List> originalImage = executeAndSnapshotPostImage(table, originalStatement, columnCount); - List> transformedImage = executeAndSnapshotPostImage(table, transformedStatement, columnCount); - - if (!originalImage.equals(transformedImage)) { - throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, originalImage, - transformedImage)); - } + List> original = snapshotSide(globalState, queries.originalStatement, queries); + List> transformed = snapshotSide(globalState, queries.transformedStatement, queries); + return new PostImages(original, transformed); } finally { - new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); + new SQLQueryAdapter(queries.dropRowIdColumn, errors, true).execute(globalState); } } @@ -261,12 +384,12 @@ private StatementPair generateInsertStatements(T table, E predicate, E transform * DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug or unexpected error * surfaces as {@link AssertionError}. * - * @param table - * the table being modified + * @param globalState + * the state whose connection the statement runs against * @param statement * the DML statement to execute - * @param columnCount - * the number of columns the post-image select returns (identifier plus content columns) + * @param queries + * supplies the transaction control and post-image select SQL and the post-image's column count * * @return the post-image, as one string list (identifier followed by content column values) per surviving row * @@ -274,20 +397,20 @@ private StatementPair generateInsertStatements(T table, E predicate, E transform * if a DBMS interaction other than running {@code statement} fails; an error from {@code statement} * itself instead surfaces as {@link IgnoreMeException} or {@link AssertionError} */ - private List> executeAndSnapshotPostImage(T table, String statement, int columnCount) + private List> snapshotSide(G globalState, String statement, ComparisonQueries queries) throws SQLException { - new SQLQueryAdapter(gen.beginTransactionStatement()).execute(state); + new SQLQueryAdapter(queries.beginTransaction).execute(globalState); try { // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. - boolean succeeded = new SQLQueryAdapter(statement, errors).execute(state); + boolean succeeded = new SQLQueryAdapter(statement, errors).execute(globalState); if (!succeeded) { // The statement hit an error the oracle tolerates; do not compare states (as EETOracle does for // SELECT). throw new IgnoreMeException(); } - return snapshotPostImage(gen.selectPostImageStatement(table), columnCount); + return snapshotPostImage(globalState, queries.selectPostImage, queries.columnCount); } finally { - new SQLQueryAdapter(gen.rollbackTransactionStatement()).execute(state); + new SQLQueryAdapter(queries.rollback).execute(globalState); } } @@ -296,6 +419,8 @@ private List> executeAndSnapshotPostImage(T table, String statement * {@code getString}). A DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug * or unexpected error surfaces as {@link AssertionError}. * + * @param globalState + * the state whose connection the select runs against * @param selectStatement * the post-image select to read; its columns are the identifier followed by the content columns * @param columnCount @@ -307,13 +432,14 @@ private List> executeAndSnapshotPostImage(T table, String statement * if cleanup fails (errors thrown elsewhere will always be rethrown as {@link IgnoreMeException} or * {@link AssertionError}) */ - private List> snapshotPostImage(String selectStatement, int columnCount) throws SQLException { + private List> snapshotPostImage(G globalState, String selectStatement, int columnCount) + throws SQLException { List> rows = new ArrayList<>(); SQLQueryAdapter q = new SQLQueryAdapter(selectStatement, errors, true, - state.getOptions().canonicalizeSqlString()); + globalState.getOptions().canonicalizeSqlString()); SQLancerResultSet result = null; try { - result = q.executeAndGet(state); + result = q.executeAndGet(globalState); if (result == null) { throw new IgnoreMeException(); } @@ -398,4 +524,9 @@ private static String renderRow(List row) { public String getLastQueryString() { return generatedQueryString; } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } } From 79e3ca09edb4d12fb0f72ce9d254c0d879d33b20 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 1 Aug 2026 14:43:06 +0800 Subject: [PATCH 132/132] Implement query reduction for EET SELECT --- src/sqlancer/Main.java | 10 + src/sqlancer/Randomly.java | 23 ++ src/sqlancer/TransformationReducer.java | 182 ++++++++++ src/sqlancer/TransformationReproducer.java | 33 ++ src/sqlancer/common/oracle/EETOracle.java | 85 ++++- .../common/oracle/EETTransformer.java | 316 +++++++++++++++--- 6 files changed, 597 insertions(+), 52 deletions(-) create mode 100644 src/sqlancer/TransformationReducer.java create mode 100644 src/sqlancer/TransformationReproducer.java diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 47ba2aedf..efd1d1d5d 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -270,6 +270,11 @@ public void setReductionContext(List> setupStatements, String bugInform this.reduceBugInformation = bugInformation; } + // for reducers that rewrite the failing queries themselves (e.g., TransformationReducer) + public void updateReducedBugInformation(String bugInformation) { + this.reduceBugInformation = bugInformation; + } + public void logReduced(StateToReproduce state) { nrReductionAttempts++; logReduced(state, "Reduction attempt " + nrReductionAttempts @@ -525,6 +530,11 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } + // reduces the oracle's transformed query itself; a no-op for reproducers whose queries are not + // built from reducible transformations + Reducer transformationReducer = new TransformationReducer<>(provider); + transformationReducer.reduce(state, reproducer, newGlobalState); + // reassemble the statements so that the main log looks like one produced // without the reducer, with the generation statements replaced by the reduced // ones and the oracle queries at the end diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 8494c189a..092744b46 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -510,6 +510,29 @@ public static double getUncachedDouble() { return getThreadRandom().get().nextDouble(); } + /** + * Computes {@code value} with this thread's random number generator temporarily replaced by a fixed-seed one, + * restoring the previous generator afterwards. This makes computations that draw randomness deterministic: for + * example, rendering an AST to SQL draws random textual variants in some DBMS implementations, and test-case + * reduction relies on re-rendering the same AST to the same string. + * + * @param + * the type of the computed value + * @param value + * the computation to run deterministically + * + * @return the computed value + */ + public static T withFixedSeedRandom(Supplier value) { + Random previousRandom = THREAD_RANDOM.get(); + THREAD_RANDOM.set(new Random(0)); + try { + return value.get(); + } finally { + THREAD_RANDOM.set(previousRandom); + } + } + public String getChar() { while (true) { String s = getString(); diff --git a/src/sqlancer/TransformationReducer.java b/src/sqlancer/TransformationReducer.java new file mode 100644 index 000000000..66af7682c --- /dev/null +++ b/src/sqlancer/TransformationReducer.java @@ -0,0 +1,182 @@ +package sqlancer; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + +import sqlancer.common.query.Query; + +/** + * Reduces the transformed query of a {@link TransformationReproducer} by disabling transformation sites, searching + * (with the same delta-debugging strategy as {@link StatementReducer}) for a minimal set of sites that still triggers + * the bug. Because each site is an individually equivalence-preserving rewrite, any subset of sites yields a + * transformed query that is still semantically equivalent to the original query, so the reduction is sound. This + * reducer runs after statement reduction, evaluating each candidate against the already-reduced database; for + * reproducers that do not implement {@link TransformationReproducer}, it does nothing. + * + * @param + * the DBMS-specific global state class + * @param + * the DBMS-specific options class + * @param + * the DBMS-specific connection class + */ +public class TransformationReducer, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> + implements Reducer { + + private final DatabaseProvider provider; + private List> statements; + private boolean observedChange; + private int partitionNum; + + private long currentReduceSteps; + private long currentReduceTime; + + private long maxReduceSteps; + private long maxReduceTime; + + private Instant timeOfReductionBegins; + + public TransformationReducer(DatabaseProvider provider) { + this.provider = provider; + } + + private boolean hasNotReachedLimit(long curr, long limit) { + if (limit == MainOptions.NO_REDUCE_LIMIT) { + return true; + } + return curr < limit; + } + + @SuppressWarnings("unchecked") + @Override + public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + if (!(reproducer instanceof TransformationReproducer)) { + return; + } + TransformationReproducer transformationReproducer = (TransformationReproducer) reproducer; + + maxReduceTime = state.getOptions().getMaxStatementReduceTime(); + maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); + + // Snapshot the (already reduced) generation statements once: createDatabase logs its setup statements + // (DROP/CREATE/USE) into the state, so the state's statement list must be reset for every candidate rather + // than read back, lest the setup statements accumulate and get re-executed mid-test-case. + statements = new ArrayList<>(); + for (Query stat : newGlobalState.getState().getStatements()) { + statements.add((Query) stat); + } + + List enabledSites = new ArrayList<>(); + for (int site = 0; site < transformationReproducer.getTransformationSiteCount(); site++) { + enabledSites.add(site); + } + // With every site disabled the transformed query renders as the original one, which cannot mismatch with + // itself, so a single remaining site cannot be reduced further. + if (enabledSites.size() < 2) { + return; + } + + timeOfReductionBegins = Instant.now(); + currentReduceSteps = 0; + currentReduceTime = 0; + partitionNum = 2; + + while (enabledSites.size() >= 2 && hasNotReachedLimit(currentReduceSteps, maxReduceSteps) + && hasNotReachedLimit(currentReduceTime, maxReduceTime)) { + observedChange = false; + + enabledSites = tryReduction(transformationReproducer, newGlobalState, enabledSites); + + if (!observedChange) { + if (partitionNum == enabledSites.size()) { + break; + } + // increase the search granularity + partitionNum = Math.min(partitionNum * 2, enabledSites.size()); + } + } + + // Leave the reproducer holding the reduced transformed query (the last candidate tried may have failed), so + // the final bug information reflects the reduction. + transformationReproducer.setEnabledTransformationSites(new HashSet<>(enabledSites)); + newGlobalState.getState().setStatements(new ArrayList<>(statements)); + newGlobalState.getLogger().updateReducedBugInformation(transformationReproducer.getBugInformation()); + newGlobalState.getLogger().logReduced(newGlobalState.getState(), + "Transformation reduction finished; the transformed query was reduced to the one shown below"); + } + + private List tryReduction(TransformationReproducer transformationReproducer, G newGlobalState, + List enabledSites) throws Exception { + + List sites = enabledSites; + + int start = 0; + int subLength = sites.size() / partitionNum; + while (start < sites.size()) { + // candidateSites = sites[:start] + sites[start+subLength:] + // in other words, remove [start, start+subLength) from sites + List candidateSites = new ArrayList<>(sites); + int endPoint = Math.min(start + subLength, candidateSites.size()); + candidateSites.subList(start, endPoint).clear(); + + if (bugStillTriggersWith(transformationReproducer, newGlobalState, candidateSites)) { + observedChange = true; + sites = candidateSites; + partitionNum = Math.max(partitionNum - 1, 2); + newGlobalState.getLogger().updateReducedBugInformation(transformationReproducer.getBugInformation()); + newGlobalState.getLogger().logReduced(newGlobalState.getState()); + break; + } + + currentReduceSteps++; + currentReduceTime = Duration.between(timeOfReductionBegins, Instant.now()).getSeconds(); + if (!hasNotReachedLimit(currentReduceSteps, maxReduceSteps) + || !hasNotReachedLimit(currentReduceTime, maxReduceTime)) { + return sites; + } + start = start + subLength; + } + return sites; + } + + /** + * Whether the bug still triggers with only {@code candidateSites} applied to the transformed query, evaluated + * against a freshly recreated database populated with the (already reduced) generation statements. + * + * @param transformationReproducer + * the reproducer whose transformed query is being reduced + * @param newGlobalState + * the state the candidate is evaluated against + * @param candidateSites + * the transformation sites to keep applied + * + * @return {@code true} if the bug still triggers with the candidate sites + */ + private boolean bugStillTriggersWith(TransformationReproducer transformationReproducer, G newGlobalState, + List candidateSites) { + transformationReproducer.setEnabledTransformationSites(new HashSet<>(candidateSites)); + try (C con2 = provider.createDatabase(newGlobalState)) { + newGlobalState.setConnection(con2); + // discard the setup statements createDatabase just logged into the state + newGlobalState.getState().setStatements(new ArrayList<>(statements)); + for (Query s : statements) { + try { + s.execute(newGlobalState); + } catch (Throwable ignoredException) { + // ignore + } + } + try { + return transformationReproducer.bugStillTriggers(newGlobalState); + } catch (Throwable ignoredException) { + // fall through: this candidate no longer triggers the bug + } + } catch (Exception e) { + e.printStackTrace(); + } + return false; + } +} diff --git a/src/sqlancer/TransformationReproducer.java b/src/sqlancer/TransformationReproducer.java new file mode 100644 index 000000000..5449af10d --- /dev/null +++ b/src/sqlancer/TransformationReproducer.java @@ -0,0 +1,33 @@ +package sqlancer; + +import java.util.Set; + +/** + * A {@link Reproducer} for bugs found by comparing an original query against a transformed one, where the transformed + * query was built by applying individually equivalence-preserving transformations (e.g. the EET rules) to the original. + * Each such application is a transformation site, identified by an index that stays stable no matter which sites are + * enabled. Disabling any subset of sites re-renders a transformed query that is still semantically equivalent to the + * original, so {@link TransformationReducer} can soundly search for a minimal set of sites that still triggers the bug. + * + * @param + * the DBMS-specific global state class + */ +public interface TransformationReproducer> extends Reproducer { + + /** + * The total number of transformation sites the transformed query was built with. This does not change when sites + * are disabled. + * + * @return the total number of transformation sites + */ + int getTransformationSiteCount(); + + /** + * Re-renders the transformed query with only the given transformation sites applied. Later + * {@link #bugStillTriggers} calls and {@link #getBugInformation} use the re-rendered query. + * + * @param enabledSites + * the indices ({@code 0} to {@code getTransformationSiteCount() - 1}) of the sites to keep applied + */ + void setEnabledTransformationSites(Set enabledSites); +} diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 0cd4ae99e..18d4e55e3 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -1,12 +1,15 @@ package sqlancer.common.oracle; import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; +import java.util.Set; import sqlancer.ComparatorHelper; +import sqlancer.Randomly; import sqlancer.Reproducer; import sqlancer.SQLGlobalState; +import sqlancer.TransformationReproducer; import sqlancer.common.ast.newast.Expression; import sqlancer.common.ast.newast.Join; import sqlancer.common.ast.newast.Select; @@ -53,13 +56,72 @@ public class EETOracle, J extends Join, E private Reproducer reproducer; private String generatedQueryString; - private final class EETReproducer extends AbstractComparisonReproducer> { + private final class EETReproducer extends AbstractComparisonReproducer> + implements TransformationReproducer { private final String originalQueryString; - private final String transformedQueryString; + // Mutable: transformation reduction re-renders the transformed query with some transformation sites disabled. + private String transformedQueryString; + private final String initialTransformedQueryString; - EETReproducer(String originalQueryString, String transformedQueryString) { + // The query parts needed to re-render the transformed query: the SELECT whose fetch columns and WHERE clause + // are replaced, the untransformed expressions, and the records of their transformations. + private final Z select; + private final List fetchColumns; + private final List fetchColumnRecords; + private final E whereClause; + private final EETTransformer.TransformationRecord whereClauseRecord; + + EETReproducer(String originalQueryString, String transformedQueryString, Z select, List fetchColumns, + List fetchColumnRecords, E whereClause, + EETTransformer.TransformationRecord whereClauseRecord) { this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; + this.initialTransformedQueryString = transformedQueryString; + this.select = select; + this.fetchColumns = fetchColumns; + this.fetchColumnRecords = fetchColumnRecords; + this.whereClause = whereClause; + this.whereClauseRecord = whereClauseRecord; + } + + @Override + public int getTransformationSiteCount() { + int siteCount = whereClauseRecord.getSiteCount(); + for (EETTransformer.TransformationRecord record : fetchColumnRecords) { + siteCount += record.getSiteCount(); + } + return siteCount; + } + + @Override + public void setEnabledTransformationSites(Set enabledSites) { + if (enabledSites.size() == getTransformationSiteCount()) { + // With every site enabled, the transformed query is the unreduced one; keep the exact string that + // originally detected the bug rather than re-rendering it (rendering an AST draws random textual + // variants, so a re-render would produce a semantically equal but untested string). + transformedQueryString = initialTransformedQueryString; + return; + } + // Pin the RNG while re-rendering so the same enabled sites always yield the same query string; the string + // tested during reduction is then exactly the string the reduced test case reports. + transformedQueryString = Randomly.withFixedSeedRandom(() -> { + // Global site indices are assigned over the fetch columns' records first (in column order), then the + // WHERE clause's record. + List replayedFetchColumns = new ArrayList<>(); + int offset = 0; + for (int i = 0; i < fetchColumns.size(); i++) { + int base = offset; + replayedFetchColumns.add(transformer.replay(fetchColumns.get(i), false, fetchColumnRecords.get(i), + site -> enabledSites.contains(base + site))); + offset += fetchColumnRecords.get(i).getSiteCount(); + } + int whereBase = offset; + E replayedWhereClause = transformer.replay(whereClause, true, whereClauseRecord, + site -> enabledSites.contains(whereBase + site)); + select.setFetchColumns(replayedFetchColumns); + select.setWhereClause(replayedWhereClause); + return select.asString(); + }); } @Override @@ -160,11 +222,17 @@ public void check() throws SQLException { } // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, - // while the WHERE clause is evaluated in a boolean context. - List transformedFetchColumns = fetchColumns.stream().map(c -> transformer.transform(c, false)) - .collect(Collectors.toList()); + // while the WHERE clause is evaluated in a boolean context. Each transformation's record is kept so the + // reproducer can replay it with transformation sites disabled during reduction. + List transformedFetchColumns = new ArrayList<>(); + List fetchColumnRecords = new ArrayList<>(); + for (E fetchColumn : fetchColumns) { + transformedFetchColumns.add(transformer.transform(fetchColumn, false)); + fetchColumnRecords.add(transformer.getLastTransformationRecord()); + } select.setFetchColumns(transformedFetchColumns); select.setWhereClause(transformer.transform(whereClause, true)); + EETTransformer.TransformationRecord whereClauseRecord = transformer.getLastTransformationRecord(); String transformedQueryString = select.asString(); List transformedResultSet; @@ -181,7 +249,8 @@ public void check() throws SQLException { // Set the reproducer before the assertion: assumeResultSetsAreEqual throws when the bug is // detected, so creating the reproducer afterwards would leave it null and prevent any reduction. - reproducer = new EETReproducer(originalQueryString, transformedQueryString); + reproducer = new EETReproducer(originalQueryString, transformedQueryString, select, fetchColumns, + fetchColumnRecords, whereClause, whereClauseRecord); ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), state); diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index b472b8aff..bb072db5a 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -2,6 +2,8 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.function.IntPredicate; import sqlancer.Randomly; import sqlancer.common.ast.newast.Expression; @@ -17,6 +19,13 @@ * ({@link #inferType} and {@link #generateExpressionOfType}) that realize the paper's {@code rand_expr(type(expr))}; * everything else (the rule logic, context threading, and tree-walking orchestration) is provided here. * + *

+ * Every {@link #transform} call records the rule applications it performs. The resulting {@link TransformationRecord} + * can later be passed to {@link #replay}, which re-applies the recorded rules with any subset of them disabled; because + * each rule application is individually equivalence-preserving, every such replay yields an expression that is still + * semantically equivalent to the input. Test-case reduction uses this to undo transformations one subset at a time + * while preserving the oracle's soundness. + * * @param * the DBMS-specific expression class * @param @@ -24,36 +33,72 @@ */ public abstract class EETTransformer, T> { - // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE - private E trueExpr() { - E p = generateBooleanExpression(); - return orExpr(orExpr(p, not(p)), isNull(p)); - } + private List> recording; // non-null while transform() is recording its rule applications + private TransformationRecord lastRecord; - // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE - private E falseExpr() { - E p = generateBooleanExpression(); - return and(and(p, not(p)), isNotNull(p)); - } + private List> replayApplications; // non-null while replay() is re-applying a record + private IntPredicate replayEnabledSites; + private int replayCursor; + private int replaySiteCursor; /** - * Implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches that of - * {@code expr}. Although the generated expression is never evaluated (it occupies the redundant branch of rules 3 - * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could - * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to - * {@code expr} itself, which trivially has the correct type (degenerating to rules 5 and 6). + * The decision made at one {@link #transformNode} call: which rule (if any) was applied at that node, together with + * the auxiliary expressions the rule drew randomly. Recording these decisions makes a transformation replayable + * with any subset of its rule applications disabled (see {@link #replay}). * - * @param expr - * the expression whose static type the generated expression must match - * - * @return a random expression whose static type matches that of {@code expr} + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain + */ + private static final class Application { + private static final Application NONE = new Application<>(null, null, null, null); + + private final Rule rule; // null when no rule was applied at this node + private final E auxiliary; // rules No. 1-4: the fresh predicate p; rules No. 5 and 6: the CASE WHEN condition + private final E deadBranch; // rules No. 3 and 4: the recorded rand_expr, or null when it degenerated to a copy + private final T deadBranchType; // rules No. 3 and 4: the inferred type deadBranch was generated for + + Application(Rule rule, E auxiliary, E deadBranch, T deadBranchType) { + this.rule = rule; + this.auxiliary = auxiliary; + this.deadBranch = deadBranch; + this.deadBranchType = deadBranchType; + } + } + + /** + * The rule applications recorded by one {@link #transform} call. The record is opaque: callers can only query the + * number of transformation sites and pass the record back to {@link #replay} on the transformer that produced it. */ - private E randExprOfSameType(E expr) { - T type = inferType(expr); - if (type == null) { - return expr; + public static final class TransformationRecord { + private final List> applications; + private final int siteCount; + + private TransformationRecord(List> applications) { + this.applications = applications; + this.siteCount = (int) applications.stream().filter(application -> application.rule != null).count(); } - return generateExpressionOfType(type); + + /** + * The number of transformation sites (rule applications) in this record. {@link #replay} numbers the sites + * {@code 0} to {@code getSiteCount() - 1} in the order they were recorded. + * + * @return the number of transformation sites + */ + public int getSiteCount() { + return siteCount; + } + } + + // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE, for any predicate p + private E trueExpr(E p) { + return orExpr(orExpr(p, not(p)), isNull(p)); + } + + // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE, for any predicate p + private E falseExpr(E p) { + return and(and(p, not(p)), isNotNull(p)); } /** @@ -61,13 +106,18 @@ private E randExprOfSameType(E expr) { * ({@link #apply}) and in which contexts it preserves the expression's value ({@link #isApplicable}). Rule No. 7 * (transform the expression to itself) is not modelled here: it is the fallback applied by {@link #applyRandomRule} * when no other rule is applicable. + * + *

+ * A rule draws no randomness of its own: the auxiliary expressions it wraps the transformed expression in come from + * the {@link Application} recorded for it, so applying a rule again during {@link EETTransformer#replay} reproduces + * the same expression. */ private enum Rule { // expr => false_expr OR expr RULE_1 { @Override - , T> E apply(EETTransformer t, E expr) { - return t.orExpr(t.falseExpr(), expr); + , T> E apply(EETTransformer t, Application application, E expr) { + return t.orExpr(t.falseExpr(application.auxiliary), expr); } @Override @@ -79,8 +129,8 @@ boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { // expr => true_expr AND expr RULE_2 { @Override - , T> E apply(EETTransformer t, E expr) { - return t.and(t.trueExpr(), expr); + , T> E apply(EETTransformer t, Application application, E expr) { + return t.and(t.trueExpr(application.auxiliary), expr); } @Override @@ -92,33 +142,43 @@ boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END RULE_3 { @Override - , T> E apply(EETTransformer t, E expr) { - return t.caseWhen(t.falseExpr(), t.randExprOfSameType(expr), expr); + , T> E apply(EETTransformer t, Application application, E expr) { + return t.caseWhen(t.falseExpr(application.auxiliary), t.deadBranch(application, expr), expr); } @Override boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { return caseWhenApplicable; } + + @Override + boolean usesDeadBranch() { + return true; + } }, // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END RULE_4 { @Override - , T> E apply(EETTransformer t, E expr) { - return t.caseWhen(t.trueExpr(), expr, t.randExprOfSameType(expr)); + , T> E apply(EETTransformer t, Application application, E expr) { + return t.caseWhen(t.trueExpr(application.auxiliary), expr, t.deadBranch(application, expr)); } @Override boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { return caseWhenApplicable; } + + @Override + boolean usesDeadBranch() { + return true; + } }, // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END RULE_5 { @Override - , T> E apply(EETTransformer t, E expr) { + , T> E apply(EETTransformer t, Application application, E expr) { // deep copy of expr is not needed, as the AST nodes are immutable anyway - return t.caseWhen(t.generateBooleanExpression(), expr, expr); + return t.caseWhen(application.auxiliary, expr, expr); } @Override @@ -129,9 +189,9 @@ boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END RULE_6 { @Override - , T> E apply(EETTransformer t, E expr) { + , T> E apply(EETTransformer t, Application application, E expr) { // deep copy of expr is not needed, as the AST nodes are immutable anyway - return t.caseWhen(t.generateBooleanExpression(), expr, expr); + return t.caseWhen(application.auxiliary, expr, expr); } @Override @@ -141,7 +201,8 @@ boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { }; /** - * Applies this rule to {@code expr}, producing a semantically equivalent expression. + * Applies this rule to {@code expr}, producing a semantically equivalent expression built from the auxiliary + * expressions {@code application} recorded for it. * * @param * the DBMS-specific expression class @@ -149,12 +210,14 @@ boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { * the DBMS-specific type domain * @param t * the transformer providing the DBMS-specific node factories + * @param application + * the recorded application of this rule, supplying its auxiliary expressions * @param expr * the expression to transform * * @return a semantically equivalent expression */ - abstract , T> E apply(EETTransformer t, E expr); + abstract , T> E apply(EETTransformer t, Application application, E expr); /** * Whether this rule preserves {@code expr}'s value in the given context. @@ -168,11 +231,22 @@ boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { * @return {@code true} if this rule preserves {@code expr}'s value in the given context */ abstract boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable); + + /** + * Whether an application of this rule carries a dead branch: an expression occupying the redundant branch of + * its CASE WHEN, which is never evaluated (see {@link EETTransformer#randomApplication}). + * + * @return {@code true} if applications of this rule carry a dead branch + */ + boolean usesDeadBranch() { + return false; + } } /** * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent - * expression. When no rule is applicable, {@code expr} is returned unchanged (rule No. 7 of the EET paper). + * expression. When no rule is applicable, {@code expr} is returned unchanged (rule No. 7 of the EET paper). The + * decision is recorded for later {@link #replay}. * * @param expr * the expression to transform @@ -190,14 +264,63 @@ protected E applyRandomRule(E expr, boolean booleanContext) { } } if (applicableRules.isEmpty()) { + record(noApplication()); return expr; // rule 7 fallback: transform expression to itself } - return Randomly.fromList(applicableRules).apply(this, expr); + Application application = randomApplication(Randomly.fromList(applicableRules), expr); + record(application); + return application.rule.apply(this, application, expr); + } + + /** + * Draws the random ingredients of one application of {@code rule} to {@code expr}. For the rules that use one, the + * dead branch implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches + * that of {@code expr}. Although that expression is never evaluated, its static type participates in the DBMS's + * CASE WHEN result-type resolution, so a type mismatch could alter the live branch's value or rendering. When the + * type of {@code expr} cannot be inferred, no dead-branch expression is generated and the rule degenerates to the + * {@code copy_expr} form of rules No. 5 and 6 (see {@link #deadBranch}). + * + * @param rule + * the transformation rule to draw an application of + * @param expr + * the expression the application will wrap + * + * @return the drawn application + */ + private Application randomApplication(Rule rule, E expr) { + if (rule.usesDeadBranch()) { + T type = inferType(expr); + E deadBranch = type == null ? null : generateExpressionOfType(type); + return new Application<>(rule, generateBooleanExpression(), deadBranch, type); + } + return new Application<>(rule, generateBooleanExpression(), null, null); + } + + /** + * The dead branch of an application of rule No. 3 or 4 around the live expression {@code expr}: the recorded random + * expression when its type still matches the type inferred for {@code expr}, and {@code expr} itself otherwise (the + * {@code copy_expr} degeneration, which trivially has the correct type). The types can stop matching during + * {@link #replay}: disabling transformation sites inside {@code expr} may change its inferred type, and reusing the + * recorded dead branch would then no longer be equivalence-preserving. + * + * @param application + * the application whose dead branch is built + * @param expr + * the live expression the application wraps + * + * @return the dead-branch expression + */ + private E deadBranch(Application application, E expr) { + if (application.deadBranch != null && Objects.equals(application.deadBranchType, inferType(expr))) { + return application.deadBranch; + } + return expr; } /** * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the - * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. + * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. The + * rule applications performed are recorded and available via {@link #getLastTransformationRecord()}. * * @param expr * the expression to transform @@ -207,11 +330,75 @@ protected E applyRandomRule(E expr, boolean booleanContext) { * @return a semantically equivalent expression */ public E transform(E expr, boolean booleanContext) { - return transformNode(expr, booleanContext, true); + recording = new ArrayList<>(); + try { + E transformed = transformNode(expr, booleanContext, true); + lastRecord = new TransformationRecord(new ArrayList<>(recording)); + return transformed; + } finally { + recording = null; + } + } + + /** + * Returns the record of the rule applications performed by the most recent {@link #transform} call, for later + * {@link #replay}. + * + * @return the most recent transformation record, or {@code null} if {@link #transform} has not been called yet + */ + public TransformationRecord getLastTransformationRecord() { + return lastRecord; + } + + /** + * Re-applies a recorded transformation to {@code expr} (the same expression that was passed to the + * {@link #transform} call that produced {@code record}), keeping only the rule applications whose site index is + * accepted by {@code enabledSites}; a disabled application leaves its subexpression untransformed. Because every + * recorded rule application is individually equivalence-preserving, the returned expression is semantically + * equivalent to {@code expr} for any subset of enabled sites, which makes replay suitable for test-case reduction: + * transformations are undone one subset at a time while the transformed query remains equivalent to the original. + * + *

+ * Replay walks the tree through the same {@link #descend} calls as the recording run, so it relies on + * {@code descend} rebuilding nodes deterministically. No new random expressions are generated: all auxiliary + * expressions are reused from the record. + * + * @param expr + * the expression the record's transform call originally transformed + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (must match the original call) + * @param record + * the record produced by this transformer's {@link #transform} call on {@code expr} + * @param enabledSites + * accepts the site indices ({@code 0} to {@code record.getSiteCount() - 1}) to keep applied + * + * @return the partially transformed expression + */ + @SuppressWarnings("unchecked") + public E replay(E expr, boolean booleanContext, TransformationRecord record, IntPredicate enabledSites) { + replayApplications = new ArrayList<>(); + for (Application application : record.applications) { + // safe: the record was produced by a transformer with the same type parameters + replayApplications.add((Application) application); + } + replayEnabledSites = enabledSites; + replayCursor = 0; + replaySiteCursor = 0; + try { + E replayed = transformNode(expr, booleanContext, true); + if (replayCursor != replayApplications.size()) { + throw new IllegalStateException("The replay visited fewer nodes than the record contains"); + } + return replayed; + } finally { + replayApplications = null; + replayEnabledSites = null; + } } /** - * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node. + * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node + * (or, during {@link #replay}, re-applies the recorded rule if its site is enabled). * * @param expr * the expression to transform @@ -224,16 +411,57 @@ public E transform(E expr, boolean booleanContext) { */ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { E descended = descend(expr, booleanContext); + if (replayApplications != null) { + return replayApplication(descended); + } if (forceApply || Randomly.getBoolean()) { return applyRandomRule(descended, booleanContext); } + record(noApplication()); return descended; } + /** + * Consumes the next recorded decision and re-applies it to the rebuilt node, unless no rule was applied there or + * the application's site is disabled. + * + * @param descended + * the rebuilt node the decision applies to + * + * @return the (possibly wrapped) node + */ + private E replayApplication(E descended) { + if (replayCursor >= replayApplications.size()) { + throw new IllegalStateException("The replay visited more nodes than the record contains"); + } + Application application = replayApplications.get(replayCursor++); + if (application.rule == null) { + return descended; + } + int site = replaySiteCursor++; + if (!replayEnabledSites.test(site)) { + return descended; + } + return application.rule.apply(this, application, descended); + } + + private void record(Application application) { + if (recording != null) { + recording.add(application); + } + } + + @SuppressWarnings("unchecked") + private Application noApplication() { + return (Application) Application.NONE; + } + /** * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each * child. Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable - * transformation will still be applied to them by the calling {@link #transformNode}. + * transformation will still be applied to them by the calling {@link #transformNode}. Implementations must be + * deterministic (in particular, visit the children of a given node in a fixed order), as {@link #replay} matches + * recorded rule applications to nodes by their visiting order. * * @param expr * the expression to descend into