diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java new file mode 100644 index 000000000..a87a635ec --- /dev/null +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -0,0 +1,51 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.oracle.EETTransformer; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like + * the other oracle generators), an EET generator creates a DBMS-specific {@link EETTransformer} that the oracle uses to + * rewrite expressions into semantically equivalent ones. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ +public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + EETGenerator setTablesAndColumns(AbstractTables tables); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + E generateBooleanExpression(); + + /** + * Creates a DBMS-specific {@link EETTransformer} backed by this generator. Called once by + * {@link sqlancer.common.oracle.EETOracle} during construction; the oracle owns the returned transformer for the + * lifetime of the test run. + * + * @return a DBMS-specific {@link EETTransformer} backed by this generator + */ + EETTransformer createTransformer(); +} diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java new file mode 100644 index 000000000..b5aedf2cb --- /dev/null +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -0,0 +1,201 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.EETGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * EET (Equivalent Expression Transformation) oracle, based on "Detecting Logic Bugs in Database Engines via Equivalent + * Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * The oracle generates a random query and then transforms its expressions (the WHERE predicate and the fetch columns) + * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation + * preserves semantics, the original and the transformed query must return the same result set; any discrepancy + * indicates a logic bug in the DBMS. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific schema class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + * @param + * the DBMS-specific global state class + */ +public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private EETGenerator gen; + private final EETTransformer transformer; + private final ExpectedErrors errors; + + private Reproducer reproducer; + private String generatedQueryString; + + private final class EETReproducer implements Reproducer { + private final String originalQueryString; + // null if the original bug was a DBMS error on the original query alone + private final String transformedQueryString; + // null if the original bug is a result set mismatch; otherwise, the message of the + // unexpected DBMS error that the original or transformed query triggered + private final String expectedErrorMessage; + + EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { + this.originalQueryString = originalQueryString; + this.transformedQueryString = transformedQueryString; + this.expectedErrorMessage = expectedErrorMessage; + } + + @Override + public boolean bugStillTriggers(G globalState) { + List originalResultSet; + List transformedResultSet; + try { + // Re-execute both queries against the current (reduced) database instead of comparing + // against a cached result set, which would be stale once statements have been removed. + originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + globalState); + if (transformedQueryString == null) { + // the original bug was a DBMS error on the original query alone, which no + // longer occurs + return false; + } + transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, + globalState); + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs + return false; + } + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, + List.of(transformedQueryString), globalState); + } catch (AssertionError resultSetMismatch) { + return true; + } + return false; + } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (transformedQueryString != null) { + sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); + } + return sb.toString(); + } + } + + public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.transformer = gen.createTransformer(); + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + reproducer = null; + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + Z select = gen.generateSelect(); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + List fetchColumns = gen.generateFetchColumns(true); + select.setFetchColumns(fetchColumns); + E whereClause = gen.generateBooleanExpression(); + select.setWhereClause(whereClause); + + String originalQueryString = select.asString(); + generatedQueryString = originalQueryString; + List originalResultSet; + try { + originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + } catch (AssertionError unexpectedError) { + // an unexpected DBMS error on the original query alone is itself a bug worth reducing; + // transformedQueryString is null because no transformed query is involved + reproducer = new EETReproducer(originalQueryString, null, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, + // while the WHERE clause is evaluated in a boolean context. + List transformedFetchColumns = fetchColumns.stream().map(c -> transformer.transform(c, false)) + .collect(Collectors.toList()); + select.setFetchColumns(transformedFetchColumns); + select.setWhereClause(transformer.transform(whereClause, true)); + + String transformedQueryString = select.asString(); + List transformedResultSet; + try { + transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, + state); + } catch (AssertionError unexpectedError) { + // the semantics-preserving transformation made the query trigger a DBMS error that the + // original did not, which is a bug worth reducing + reproducer = new EETReproducer(originalQueryString, transformedQueryString, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + // Set the reproducer before the assertion: assumeResultSetsAreEqual throws when the bug is + // detected, so creating the reproducer afterwards would leave it null and prevent any reduction. + reproducer = new EETReproducer(originalQueryString, transformedQueryString, null); + + ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, + List.of(transformedQueryString), state); + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java new file mode 100644 index 000000000..b472b8aff --- /dev/null +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -0,0 +1,360 @@ +package sqlancer.common.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; + +/** + * Abstract base class for EET (Equivalent Expression Transformation) tree-walkers, based on "Detecting Logic Bugs in + * Database Engines via Equivalent Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework + * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST + * nodes from their transformed children, the abstract factory methods to construct new nodes, and the type hooks + * ({@link #inferType} and {@link #generateExpressionOfType}) that realize the paper's {@code rand_expr(type(expr))}; + * everything else (the rule logic, context threading, and tree-walking orchestration) is provided here. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain used by {@link #inferType} and {@link #generateExpressionOfType} + */ +public abstract class EETTransformer, T> { + + // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE + private E trueExpr() { + E p = generateBooleanExpression(); + return orExpr(orExpr(p, not(p)), isNull(p)); + } + + // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE + private E falseExpr() { + E p = generateBooleanExpression(); + return and(and(p, not(p)), isNotNull(p)); + } + + /** + * Implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches that of + * {@code expr}. Although the generated expression is never evaluated (it occupies the redundant branch of rules 3 + * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could + * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to + * {@code expr} itself, which trivially has the correct type (degenerating to rules 5 and 6). + * + * @param expr + * the expression whose static type the generated expression must match + * + * @return a random expression whose static type matches that of {@code expr} + */ + private E randExprOfSameType(E expr) { + T type = inferType(expr); + if (type == null) { + return expr; + } + return generateExpressionOfType(type); + } + + /** + * The first six transformation rules of the EET paper (Table 2). Each rule knows how to apply itself + * ({@link #apply}) and in which contexts it preserves the expression's value ({@link #isApplicable}). Rule No. 7 + * (transform the expression to itself) is not modelled here: it is the fallback applied by {@link #applyRandomRule} + * when no other rule is applicable. + */ + private enum Rule { + // expr => false_expr OR expr + RULE_1 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.orExpr(t.falseExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => true_expr AND expr + RULE_2 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.and(t.trueExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END + RULE_3 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.falseExpr(), t.randExprOfSameType(expr), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END + RULE_4 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.trueExpr(), expr, t.randExprOfSameType(expr)); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END + RULE_5 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END + RULE_6 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }; + + /** + * Applies this rule to {@code expr}, producing a semantically equivalent expression. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain + * @param t + * the transformer providing the DBMS-specific node factories + * @param expr + * the expression to transform + * + * @return a semantically equivalent expression + */ + abstract , T> E apply(EETTransformer t, E expr); + + /** + * Whether this rule preserves {@code expr}'s value in the given context. + * + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (rules 1 and 2 are only applicable if + * this is the case) + * @param caseWhenApplicable + * whether {@code expr} may be wrapped in a CASE WHEN expression + * + * @return {@code true} if this rule preserves {@code expr}'s value in the given context + */ + abstract boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable); + } + + /** + * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent + * expression. When no rule is applicable, {@code expr} is returned unchanged (rule No. 7 of the EET paper). + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * + * @return a semantically equivalent expression + */ + protected E applyRandomRule(E expr, boolean booleanContext) { + boolean caseWhenApplicable = isCaseWhenApplicable(expr); + List applicableRules = new ArrayList<>(); + for (Rule rule : Rule.values()) { + if (rule.isApplicable(booleanContext, caseWhenApplicable)) { + applicableRules.add(rule); + } + } + if (applicableRules.isEmpty()) { + return expr; // rule 7 fallback: transform expression to itself + } + return Randomly.fromList(applicableRules).apply(this, expr); + } + + /** + * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the + * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * + * @return a semantically equivalent expression + */ + public E transform(E expr, boolean booleanContext) { + return transformNode(expr, booleanContext, true); + } + + /** + * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * @param forceApply + * whether a rule must be applied at this node rather than only with some probability + * + * @return the transformed expression + */ + protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { + E descended = descend(expr, booleanContext); + if (forceApply || Randomly.getBoolean()) { + return applyRandomRule(descended, booleanContext); + } + return descended; + } + + /** + * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each + * child. Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable + * transformation will still be applied to them by the calling {@link #transformNode}. + * + * @param expr + * the expression to descend into + * @param booleanContext + * the context in which {@code expr} itself is evaluated (used to determine child contexts) + * + * @return a rebuilt copy of {@code expr} with transformed children, or {@code expr} itself if it is a leaf + */ + protected abstract E descend(E expr, boolean booleanContext); + + /** + * Builds {@code left AND right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left AND right} expression + */ + protected abstract E and(E left, E right); + + /** + * Builds {@code left OR right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left OR right} expression + */ + protected abstract E orExpr(E left, E right); + + /** + * Builds {@code NOT expr}. + * + * @param expr + * the operand + * + * @return the {@code NOT expr} expression + */ + protected abstract E not(E expr); + + /** + * Builds {@code expr IS NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NULL} expression + */ + protected abstract E isNull(E expr); + + /** + * Builds {@code expr IS NOT NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NOT NULL} expression + */ + protected abstract E isNotNull(E expr); + + /** + * Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. + * + * @param condition + * the WHEN condition + * @param thenExpr + * the THEN branch + * @param elseExpr + * the ELSE branch + * + * @return the CASE WHEN expression + */ + protected abstract E caseWhen(E condition, E thenExpr, E elseExpr); + + /** + * Generates a fresh random boolean expression, reusing the variables available to the query generator. + * + * @return a fresh random boolean expression + */ + protected abstract E generateBooleanExpression(); + + /** + * Infers the static type of {@code expr}, or returns {@code null} if it cannot be determined. The type domain + * {@code T} is DBMS-specific and may be coarse: it only needs to be precise enough that replacing an expression + * with another of the same {@code T} leaves the DBMS's CASE WHEN result-type resolution unaffected. Returning + * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead branch. + * Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is + * uncertain. + * + * @param expr + * the expression whose static type is to be inferred + * + * @return the inferred static type of {@code expr}, or {@code null} if it cannot be determined + */ + protected abstract T inferType(E expr); + + /** + * Generates a fresh random expression of static type {@code type}, reusing the variables available to the query + * generator. DBMSs with a typed expression generator can delegate to it directly; DBMSs with an untyped generator + * can instead wrap an arbitrary random expression in a CAST to {@code type} (which requires every value of + * {@code T} to be a valid CAST target). + * + * @param type + * the static type the generated expression must have + * + * @return a fresh random expression of static type {@code type} + */ + protected abstract E generateExpressionOfType(T type); + + /** + * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not + * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + * + * @param expr + * the expression to test + * + * @return {@code true} if {@code expr} may be wrapped in a CASE WHEN expression + */ + protected abstract boolean isCaseWhenApplicable(E expr); +} diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index 83e08677a..ed5ddc489 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -5,6 +5,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.EETOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; @@ -82,5 +83,14 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLDQEOracle(globalState); } + }, + EET { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + return new EETOracle<>(globalState, gen, expectedErrors); + } }; } diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index 8ae783fa0..b71d0498c 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -6,11 +6,14 @@ public class MySQLCastOperation implements MySQLExpression { private final CastType type; public enum CastType { - SIGNED, UNSIGNED; + SIGNED, UNSIGNED, + // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never + // evaluated, so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + CHAR, FLOAT, DOUBLE, DECIMAL; public static CastType getRandom() { return SIGNED; - // return Randomly.fromOptions(CastType.values()); + // return Randomly.fromOptions(CastType.SIGNED, CastType.UNSIGNED); } } diff --git a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java index c87048a4b..45a1fe888 100644 --- a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java +++ b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java @@ -62,6 +62,10 @@ public MySQLUnaryPrefixOperation(MySQLExpression expr, MySQLUnaryPrefixOperator super(expr, op); } + public MySQLUnaryPrefixOperator getOp() { + return op; + } + @Override public MySQLConstant getExpectedValue() { MySQLConstant subExprVal = expr.getExpectedValue(); diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index d8ce5dd37..da304ac67 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -9,8 +9,10 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTables; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; @@ -45,10 +47,12 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, - CERTGenerator { + CERTGenerator, + EETGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; @@ -218,6 +222,22 @@ public List generateOrderBys() { return newOrderBys; } + public MySQLAggregate generateAggregate() { + MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); + + if (func.isVariadic()) { + int nrExprs = Randomly.smallNumber() + 1; + List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) + .collect(Collectors.toList()); + + return new MySQLAggregate(exprs, func); + } else { + return new MySQLAggregate(List.of(generateExpression()), func); + } + } + + // --- Shared oracle infrastructure (TLPWhere / CERT / EET) --- + @Override public MySQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); @@ -251,6 +271,8 @@ public List generateFetchColumns(boolean shouldCreateDummy) { return columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); } + // --- CERT oracle --- + @Override public String generateExplainQuery(MySQLSelect select) { return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed @@ -258,20 +280,6 @@ public String generateExplainQuery(MySQLSelect select) { // now be specified } - public MySQLAggregate generateAggregate() { - MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); - - if (func.isVariadic()) { - int nrExprs = Randomly.smallNumber() + 1; - List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) - .collect(Collectors.toList()); - - return new MySQLAggregate(exprs, func); - } else { - return new MySQLAggregate(List.of(generateExpression()), func); - } - } - @Override public boolean mutate(MySQLSelect select) { List> mutators = new ArrayList<>(); @@ -355,4 +363,11 @@ boolean mutateOr(MySQLSelect select) { return true; } } + + // --- EET oracle --- + + @Override + public EETTransformer createTransformer() { + return new MySQLEETTransformer(this); + } } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 054a66cb6..40e325041 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -362,14 +362,19 @@ private void appendType(MySQLDataType randomType) { } if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS - || o == MySQLOracleFactory.DQP)) { + || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET)) { sb.append(" ZEROFILL"); } } } - public static void optionallyAddPrecisionAndScale(StringBuilder sb) { - if (Randomly.getBoolean() && !MySQLBugs.bug99183) { + private void optionallyAddPrecisionAndScale(StringBuilder sb) { + // The EET oracle's type inference assumes FLOAT/DOUBLE/DECIMAL columns are created without (M, D) (see + // MySQLEETTransformer#inferColumnType), so precision/scale is omitted while EET is active. This restriction can + // be lifted once (M, D) is tracked through the codebase and reflected in the CAST target types. + boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.EET); + if (Randomly.getBoolean() && !MySQLBugs.bug99183 && !eetActive) { sb.append("("); // The maximum number of digits (M) for DECIMAL is 65 long m = Randomly.getNotCachedInteger(1, 65); diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java new file mode 100644 index 000000000..2e4394eec --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -0,0 +1,287 @@ +package sqlancer.mysql.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.oracle.EETTransformer; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLBetweenOperation; +import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; +import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLCastOperation; +import sqlancer.mysql.ast.MySQLCastOperation.CastType; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLComputableFunction; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExists; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + +/** + * MySQL implementation of the {@link EETTransformer EET} tree-walker. Implements {@link #descend} to rebuild MySQL AST + * nodes from their transformed children, threading the correct boolean/scalar context into each child. + * + *

+ * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target + * types ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning + * {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping + * it in a CAST. + */ +public class MySQLEETTransformer extends EETTransformer { + + private static final boolean BOOLEAN = true; + private static final boolean SCALAR = false; + + private final MySQLExpressionGenerator gen; + + public MySQLEETTransformer(MySQLExpressionGenerator gen) { + this.gen = gen; + } + + @Override + protected MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { + if (expr instanceof MySQLBinaryLogicalOperation) { + // AND/OR/XOR: both operands are evaluated in a boolean context. + MySQLBinaryLogicalOperation op = (MySQLBinaryLogicalOperation) expr; + return new MySQLBinaryLogicalOperation(transformNode(op.getLeft(), BOOLEAN, false), + transformNode(op.getRight(), BOOLEAN, false), op.getOp()); + } else if (expr instanceof MySQLBinaryComparisonOperation) { + MySQLBinaryComparisonOperation op = (MySQLBinaryComparisonOperation) expr; + return new MySQLBinaryComparisonOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLBinaryOperation) { + MySQLBinaryOperation op = (MySQLBinaryOperation) expr; + return new MySQLBinaryOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + MySQLUnaryPrefixOperation op = (MySQLUnaryPrefixOperation) expr; + boolean childContext = op.getOp() == MySQLUnaryPrefixOperator.NOT ? BOOLEAN : SCALAR; + return new MySQLUnaryPrefixOperation(transformNode(op.getExpression(), childContext, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPostfixOperation) { + // The operand is transformed value-preservingly (scalar), which is safe for IS NULL/IS TRUE/IS FALSE. + MySQLUnaryPostfixOperation op = (MySQLUnaryPostfixOperation) expr; + return new MySQLUnaryPostfixOperation(transformNode(op.getExpression(), SCALAR, false), op.getOperator(), + op.isNegated()); + } else if (expr instanceof MySQLCastOperation) { + MySQLCastOperation op = (MySQLCastOperation) expr; + return new MySQLCastOperation(transformNode(op.getExpr(), SCALAR, false), op.getType()); + } else if (expr instanceof MySQLBetweenOperation) { + MySQLBetweenOperation op = (MySQLBetweenOperation) expr; + return new MySQLBetweenOperation(transformNode(op.getExpr(), SCALAR, false), + transformNode(op.getLeft(), SCALAR, false), transformNode(op.getRight(), SCALAR, false)); + } else if (expr instanceof MySQLInOperation) { + MySQLInOperation op = (MySQLInOperation) expr; + List listElements = op.getListElements().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); + return new MySQLInOperation(transformNode(op.getExpr(), SCALAR, false), listElements, op.isTrue()); + } else if (expr instanceof MySQLComputableFunction) { + MySQLComputableFunction op = (MySQLComputableFunction) expr; + MySQLExpression[] args = op.getArguments(); + MySQLExpression[] newArgs = new MySQLExpression[args.length]; + for (int i = 0; i < args.length; i++) { + newArgs[i] = transformNode(args[i], SCALAR, false); + } + return new MySQLComputableFunction(op.getFunction(), newArgs); + } else if (expr instanceof MySQLCaseOperator) { + return descendCase((MySQLCaseOperator) expr); + } + return expr; + } + + private MySQLExpression descendCase(MySQLCaseOperator caseOp) { + MySQLExpression switchCondition = caseOp.getSwitchCondition(); + // Without a switch operand the WHEN conditions are boolean; with one they are compared against the operand. + boolean conditionContext = switchCondition == null ? BOOLEAN : SCALAR; + MySQLExpression newSwitch = switchCondition == null ? null : transformNode(switchCondition, SCALAR, false); + List conditions = caseOp.getConditions().stream() + .map(e -> transformNode(e, conditionContext, false)).collect(Collectors.toList()); + List expressions = caseOp.getExpressions().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); + MySQLExpression elseExpr = caseOp.getElseExpr() == null ? null + : transformNode(caseOp.getElseExpr(), SCALAR, false); + return new MySQLCaseOperator(newSwitch, conditions, expressions, elseExpr); + } + + @Override + protected MySQLExpression and(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); + } + + @Override + protected MySQLExpression orExpr(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); + } + + @Override + protected MySQLExpression not(MySQLExpression expr) { + return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); + } + + @Override + protected MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); + } + + @Override + protected MySQLExpression isNotNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); + } + + @Override + protected MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { + return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); + } + + @Override + protected MySQLExpression generateBooleanExpression() { + return gen.generateBooleanExpression(); + } + + @Override + protected MySQLExpression generateExpressionOfType(CastType type) { + // The MySQL expression generator is untyped, so the type of an arbitrary random expression is pinned by + // wrapping it in a CAST to the requested type. + return new MySQLCastOperation(gen.generateExpression(), type); + } + + @Override + protected CastType inferType(MySQLExpression expr) { + if (expr instanceof MySQLBinaryLogicalOperation || expr instanceof MySQLBinaryComparisonOperation + || expr instanceof MySQLUnaryPostfixOperation || expr instanceof MySQLBetweenOperation + || expr instanceof MySQLInOperation || expr instanceof MySQLExists) { + // Predicates evaluate to the boolean values 0/1, which are signed BIGINT. + return CastType.SIGNED; + } else if (expr instanceof MySQLBinaryOperation) { + // The bit operators &, | and ^ return BIGINT UNSIGNED. + return CastType.UNSIGNED; + } else if (expr instanceof MySQLCastOperation) { + return ((MySQLCastOperation) expr).getType(); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + return inferUnaryPrefixType((MySQLUnaryPrefixOperation) expr); + } else if (expr instanceof MySQLConstant) { + return inferConstantType((MySQLConstant) expr); + } else if (expr instanceof MySQLColumnReference) { + return inferColumnType((MySQLColumnReference) expr); + } else if (expr instanceof MySQLComputableFunction) { + return inferFunctionType((MySQLComputableFunction) expr); + } else if (expr instanceof MySQLCaseOperator) { + return inferCaseType((MySQLCaseOperator) expr); + } + return null; + } + + private CastType inferUnaryPrefixType(MySQLUnaryPrefixOperation op) { + if (op.getOp() == MySQLUnaryPrefixOperator.NOT) { + return CastType.SIGNED; + } + CastType operandType = inferType(op.getExpression()); + if (op.getOp() == MySQLUnaryPrefixOperator.PLUS) { + return operandType; + } + if (op.getOp() == MySQLUnaryPrefixOperator.MINUS) { + if (operandType == CastType.UNSIGNED) { + return CastType.SIGNED; + } else if (operandType == CastType.FLOAT) { + return CastType.DOUBLE; + } else if (operandType != CastType.CHAR) { + return operandType; + } + } + return null; + } + + private CastType inferConstantType(MySQLConstant constant) { + if (constant instanceof MySQLConstant.MySQLIntConstant) { + return constant.isSigned() ? CastType.SIGNED : CastType.UNSIGNED; + } else if (constant instanceof MySQLConstant.MySQLTextConstant) { + return CastType.CHAR; + } else if (constant instanceof MySQLConstant.MySQLDoubleConstant) { + return CastType.DOUBLE; + } + return null; // the NULL constant has no type of its own + } + + private CastType inferColumnType(MySQLColumnReference ref) { + switch (ref.getColumn().getType()) { + case INT: + return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns + case VARCHAR: + return CastType.CHAR; + // FLOAT/DOUBLE/DECIMAL columns are created without (M, D) while EET is active, so the plain + // CAST target below matches the column's type. Reintroducing (M, D) for better coverage would + // require tracking it here and emitting the exact precision/scale in the CAST. + case FLOAT: + return CastType.FLOAT; + case DOUBLE: + return CastType.DOUBLE; + case DECIMAL: + return CastType.DECIMAL; + default: + return null; + } + } + + private CastType inferFunctionType(MySQLComputableFunction func) { + MySQLExpression[] args = func.getArguments(); + switch (func.getFunction()) { + case BIT_COUNT: + return CastType.SIGNED; + case IF: + // The result type aggregates the types of the two value arguments (the condition does not contribute). + return commonType(args[1], args[2]); + case COALESCE: + case IFNULL: + case LEAST: + case GREATEST: + return commonType(args); + default: + return null; + } + } + + private CastType inferCaseType(MySQLCaseOperator caseOp) { + List branches = new ArrayList<>(caseOp.getExpressions()); + if (caseOp.getElseExpr() != null) { + branches.add(caseOp.getElseExpr()); + } + return commonType(branches.toArray(new MySQLExpression[0])); + } + + /** + * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same + * inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * + * @param exprs + * the result-type-determining subexpressions + * + * @return the common inferred type of {@code exprs}, or {@code null} if they do not share one + */ + private CastType commonType(MySQLExpression... exprs) { + CastType common = null; + for (MySQLExpression expr : exprs) { + CastType type = inferType(expr); + if (type == null || common != null && type != common) { + return null; + } + common = type; + } + return common; + } + + @Override + protected boolean isCaseWhenApplicable(MySQLExpression expr) { + // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET + // paper); aggregates are excluded to avoid placing them in invalid contexts. + return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); + } +}