From 4cf31d15f2f841686365f0167ad489222eb4b90f Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 13 Aug 2026 13:40:00 +0800 Subject: [PATCH 1/3] Implement INSERT support for EET DML oracle --- src/sqlancer/common/gen/EETDMLGenerator.java | 72 +++++++++++++++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 61 ++++++++++++---- .../mysql/gen/MySQLExpressionGenerator.java | 16 +++++ 3 files changed, 130 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 0c84901f3..c10c59b5a 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -19,9 +19,9 @@ *

* Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run - * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are - * produced. The resulting state is compared as a full post-image (each surviving row's identifier and content column - * values), which covers every DML statement: a DELETE removes rows from it, an UPDATE changes values in it. + * (each in a rolled-back transaction), so both executions observe the same identifiers. The resulting state is compared + * as a full post-image (each surviving row's identifier and content column values), which covers any of the three DML + * statements (DELETE, UPDATE, INSERT). * *

* Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. @@ -66,6 +66,15 @@ public interface EETDMLGenerator, T extends AbstractTabl */ List> generateSetAssignments(); + /** + * Generates a fresh value expression for each content column of the current table, used as an INSERT statement's + * inserted values. The returned expressions are positionally aligned with {@link AbstractTable#getColumns()}, and + * each is transformed by the oracle. + * + * @return one fresh random value expression per content column, in {@link AbstractTable#getColumns()} order + */ + List generateInsertValues(); + /** * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's * expressions into semantically equivalent ones. @@ -106,6 +115,17 @@ public interface EETDMLGenerator, T extends AbstractTabl */ String rowIdColumnType(); + /** + * A SQL expression, evaluated once per source row of an {@code INSERT ... SELECT}, that derives the inserted row's + * {@link #ROW_ID_COLUMN} value from the source row's identifier. It must be deterministic (so both the original and + * transformed statements assign the same identifiers), unique per source row, and distinct from every existing + * identifier (so an inserted row never collides with the source row it was derived from in the post-image). DBMS- + * specific because it names a suitable derivation function (e.g. a hash of the source identifier). + * + * @return the SQL expression deriving an inserted row's identifier from the source row's {@link #ROW_ID_COLUMN} + */ + String insertedRowIdExpression(); + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- /** @@ -139,8 +159,9 @@ default String dropRowIdColumnStatement(T table) { * *

* This single value-level snapshot is the comparison surface for all DML statements: a DELETE removes rows from it, - * an UPDATE changes column values in it. Row identity alone (which the identifier already captures) would suffice - * for DELETE, but not for UPDATE, where the two runs could touch the same rows yet write different values. + * an UPDATE changes column values in it, an INSERT adds rows to it. Row identity alone (which the identifier + * already captures) would suffice for DELETE, but not for UPDATE, where the two runs could touch the same rows yet + * write different values. * * @param table * the table to snapshot @@ -222,6 +243,47 @@ default String updateStatement(T table, List> assignments, E pre + orderByLimitClause(orderByColumns, limit); } + /** + * SQL that inserts a new row into {@code table} for each source row (optionally filtered by {@code predicate}), + * setting each content column to its corresponding value in {@code values}. + * + *

+ * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value + * expressions reference the table's columns (the transformer injects column references into its equivalent + * sub-expressions), which are legal in a {@code SELECT} but not in a {@code VALUES} clause. Each inserted row's + * {@link #ROW_ID_COLUMN} is derived from its source row via {@link #insertedRowIdExpression()}, giving it a + * deterministic identifier that is unique and distinct from every existing one, so the two statements' post-images + * align (and inserted rows never collide with their source rows). + * + * @param table + * the table to insert into + * @param values + * one value expression per content column, positionally aligned with {@link AbstractTable#getColumns()}; + * each is rendered via {@link #asString} + * @param predicate + * the WHERE predicate filtering the source rows, or {@code null} to insert from every source row; + * rendered via {@link #asString} + * + * @return the SQL statement + */ + default String insertStatement(T table, List values, E predicate) { + List columnNames = new ArrayList<>(); + columnNames.add(ROW_ID_COLUMN); + List selectItems = new ArrayList<>(); + selectItems.add(insertedRowIdExpression()); + List columns = table.getColumns(); + for (int i = 0; i < columns.size(); i++) { + columnNames.add(columns.get(i).getName()); + selectItems.add(asString(values.get(i))); + } + String statement = "INSERT INTO " + table.getName() + " (" + String.join(", ", columnNames) + ") SELECT " + + String.join(", ", selectItems) + " FROM " + table.getName(); + if (predicate != null) { + statement += " WHERE " + asString(predicate); + } + return statement; + } + /** * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and * {@link #updateStatement}, or the empty string when {@code limit} is null. diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 4731c04ed..ba9b9d6c7 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -9,6 +9,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import java.util.function.Supplier; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -37,13 +38,15 @@ * statements can be compared against the same starting state without permanently modifying the database. The state is * captured as a full post-image: each surviving row's identifier together with its content column values, ordered by * the identifier. This single value-level surface covers every DML statement — a DELETE removes rows from it, an UPDATE - * changes values in it (row identity alone would suffice for DELETE, but not for UPDATE, which also transforms the - * written values). Because rolling back a statement requires a transactional storage engine, the DBMS-specific setup - * must ensure only such engines are used while this oracle is active. + * changes values in it, an INSERT adds rows to it (row identity alone would suffice for DELETE, but not for UPDATE, + * which also transforms the written values). Because rolling back a statement requires a transactional storage engine, + * the DBMS-specific setup must ensure only such engines are used while this oracle is active. * *

- * DELETE and UPDATE are currently supported (one is chosen at random per check). Statement reduction is not yet - * implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database + * DELETE, UPDATE and INSERT are currently supported (one is chosen at random per check). INSERT uses the + * {@code INSERT ... SELECT} form so its transformed value expressions may reference columns; each inserted row is given + * a deterministic identifier derived from its source row so the two runs' post-images align. Statement reduction is not + * yet implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database * reduction. * * @param @@ -97,16 +100,16 @@ public void check() throws SQLException { // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. - Integer limit = null; - List orderByColumns = List.of(); - if (Randomly.getBoolean()) { - limit = (int) Randomly.getNotCachedInteger(0, 10); - orderByColumns = Randomly.subset(table.getColumns()); - } + boolean withLimit = Randomly.getBoolean(); + Integer limit = withLimit ? (int) Randomly.getNotCachedInteger(0, 10) : null; + List orderByColumns = withLimit ? Randomly.subset(table.getColumns()) : List.of(); - StatementPair statements = Randomly.getBoolean() - ? generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit) - : generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit); + // Generators for the different kinds of statement this oracle supports. One is chosen at random per check + List> statementGenerators = List.of( + () -> generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit), + () -> generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit), + () -> generateInsertStatements(table, predicate, transformedPredicate)); + StatementPair statements = Randomly.fromList(statementGenerators).get(); String originalStatement = statements.original; String transformedStatement = statements.transformed; generatedQueryString = originalStatement; @@ -201,6 +204,36 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform gen.deleteStatement(table, transformedPredicate, orderByColumns, limit)); } + /** + * Generates an {@code INSERT ... SELECT} and its transformed counterpart. Besides the WHERE predicate, which + * filters the source rows and is optional here, INSERT also transforms each inserted value in a scalar context. + * + *

+ * Unlike DELETE and UPDATE, no limit is applied: {@link EETDMLGenerator#insertStatement} renders no ordering or + * limit, so one row is inserted per source row the predicate keeps. Nothing about INSERT rules a limit out — its + * source SELECT could carry the same ordering and limit the other statement kinds use, and the two runs would still + * read the same source rows — it is just not generated. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate) { + List values = gen.generateInsertValues(); + List transformedValues = new ArrayList<>(); + for (E value : values) { + transformedValues.add(transformer.transform(value, false)); + } + boolean withPredicate = Randomly.getBoolean(); + return new StatementPair(gen.insertStatement(table, values, withPredicate ? predicate : null), + gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null)); + } + /** * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9fbaef12a..baea11f65 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -268,6 +268,14 @@ public List> generateSetAssignments() { return assignments; } + @Override + public List generateInsertValues() { + // One value per content column, in schema order (aligned with the INSERT column list). As with the normal + // INSERT workload, each value is an arbitrary expression (not type-matched to the column); any resulting + // type/range/constraint error is on the oracle's expected-error allow-list. + return columns.stream().map(c -> generateExpression()).collect(Collectors.toList()); + } + @Override public MySQLSelect generateSelect() { return new MySQLSelect(); @@ -408,4 +416,12 @@ public String rowIdColumnType() { // Holds a 36-character UUID string produced by stampRowIdsStatement. return "VARCHAR(36)"; } + + @Override + public String insertedRowIdExpression() { + // The source row's identifier with its dashes removed: deterministic (identical across both runs) and unique + // per + // source row. Fits the identifier column's VARCHAR(36). + return String.format("REPLACE(%s, '-', '')", ROW_ID_COLUMN); + } } From 7ead2e762106cdab3c71be9b094b1c42b033afb7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 13 Aug 2026 13:50:17 +0800 Subject: [PATCH 2/3] Add support for LIMIT on source rows of INSERT ... SELECT statements in EET --- src/sqlancer/common/gen/EETDMLGenerator.java | 16 ++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 53 ++++++++++++++------ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index c10c59b5a..557220d06 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -245,7 +245,8 @@ default String updateStatement(T table, List> assignments, E pre /** * SQL that inserts a new row into {@code table} for each source row (optionally filtered by {@code predicate}), - * setting each content column to its corresponding value in {@code values}. + * setting each content column to its corresponding value in {@code values}, optionally limited to the first + * {@code limit} source rows (see {@link #orderByLimitClause}). * *

* The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value @@ -263,10 +264,15 @@ default String updateStatement(T table, List> assignments, E pre * @param predicate * the WHERE predicate filtering the source rows, or {@code null} to insert from every source row; * rendered via {@link #asString} + * @param orderByColumns + * the columns to order the source rows by before the row-id tiebreaker (may be empty); only used when + * {@code limit} is non-null + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit * * @return the SQL statement */ - default String insertStatement(T table, List values, E predicate) { + default String insertStatement(T table, List values, E predicate, List orderByColumns, Integer limit) { List columnNames = new ArrayList<>(); columnNames.add(ROW_ID_COLUMN); List selectItems = new ArrayList<>(); @@ -281,12 +287,12 @@ default String insertStatement(T table, List values, E predicate) { if (predicate != null) { statement += " WHERE " + asString(predicate); } - return statement; + return statement + orderByLimitClause(orderByColumns, limit); } /** - * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and - * {@link #updateStatement}, or the empty string when {@code limit} is null. + * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement}, + * {@link #updateStatement} and {@link #insertStatement}, or the empty string when {@code limit} is null. * *

* The rows are ordered by {@code orderByColumns} followed by {@link #ROW_ID_COLUMN} as a tiebreaker. Because the diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index ba9b9d6c7..02bff8165 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -9,7 +9,6 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; -import java.util.function.Supplier; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -100,16 +99,18 @@ public void check() throws SQLException { // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. - boolean withLimit = Randomly.getBoolean(); - Integer limit = withLimit ? (int) Randomly.getNotCachedInteger(0, 10) : null; - List orderByColumns = withLimit ? Randomly.subset(table.getColumns()) : List.of(); + Integer limit = null; + List orderByColumns = List.of(); + if (Randomly.getBoolean()) { + limit = (int) Randomly.getNotCachedInteger(0, 10); + orderByColumns = Randomly.subset(table.getColumns()); + } // Generators for the different kinds of statement this oracle supports. One is chosen at random per check - List> statementGenerators = List.of( - () -> generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit), - () -> generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit), - () -> generateInsertStatements(table, predicate, transformedPredicate)); - StatementPair statements = Randomly.fromList(statementGenerators).get(); + List> statementGenerators = List.of(this::generateDeleteStatements, + this::generateUpdateStatements, this::generateInsertStatements); + StatementPair statements = Randomly.fromList(statementGenerators).generate(table, predicate, + transformedPredicate, orderByColumns, limit); String originalStatement = statements.original; String transformedStatement = statements.transformed; generatedQueryString = originalStatement; @@ -140,6 +141,22 @@ public void check() throws SQLException { } } + /** + * Generates a DML statement of one kind together with its transformed counterpart. The kinds share this signature + * so the oracle can pick one of them at random per check. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ + @FunctionalInterface + private interface DMLStatementGenerator { + StatementPair generate(T table, E predicate, E transformedPredicate, List orderByColumns, Integer limit); + } + /** * A DML statement and its transformed counterpart, which must leave the database in the same state. */ @@ -209,10 +226,7 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform * filters the source rows and is optional here, INSERT also transforms each inserted value in a scalar context. * *

- * Unlike DELETE and UPDATE, no limit is applied: {@link EETDMLGenerator#insertStatement} renders no ordering or - * limit, so one row is inserted per source row the predicate keeps. Nothing about INSERT rules a limit out — its - * source SELECT could carry the same ordering and limit the other statement kinds use, and the two runs would still - * read the same source rows — it is just not generated. + * The ordering and limit cap the source rows the statement reads, so it inserts one row per source row kept. * * @param table * the table being modified @@ -220,18 +234,25 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform * the WHERE predicate of the original statement * @param transformedPredicate * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the source rows, empty if the statement is not capped by a limit + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit * * @return the original statement together with its transformed counterpart */ - private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate) { + private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { List values = gen.generateInsertValues(); List transformedValues = new ArrayList<>(); for (E value : values) { transformedValues.add(transformer.transform(value, false)); } boolean withPredicate = Randomly.getBoolean(); - return new StatementPair(gen.insertStatement(table, values, withPredicate ? predicate : null), - gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null)); + return new StatementPair( + gen.insertStatement(table, values, withPredicate ? predicate : null, orderByColumns, limit), + gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null, + orderByColumns, limit)); } /** From f46934ecb6ec699c0315631e38e71a8e97e8b27f Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 10:11:10 +0800 Subject: [PATCH 3/3] Improve robustness of MySQL EET DML by ensuring InnoDB engine is always chosen --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 0746056ad..d27d6e681 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -164,7 +164,15 @@ public static List getRandomTableOptions() { } private void appendTableOptions() { - List tableOptions = TableOptions.getRandomTableOptions(); + List tableOptions = new ArrayList<>(TableOptions.getRandomTableOptions()); + // The EET DML oracle rolls back each statement to compare database states, which requires a transactional + // engine. The ENGINE option already forces InnoDB when the oracle is active (see the ENGINE case below), but it + // is only emitted when randomly chosen; otherwise the table would inherit the server's default engine, which is + // not guaranteed transactional. Force the option to always be present so the engine is never left to the + // server default. + if (globalState.usesEETDML() && !tableOptions.contains(TableOptions.ENGINE)) { + tableOptions.add(TableOptions.ENGINE); + } int i = 0; for (TableOptions o : tableOptions) { if (i++ != 0) {